# Mininet-containernet6 (NAT DR_ HDR proxcy)
[TOC]
# Lvs-NAT
```
apt install ipvsadm
```
```
apt install haproxy
```
## script 1
```
#!/usr/bin/env python
from mininet.net import Containernet
from mininet.cli import CLI
from mininet.link import Link,TCLink,Intf
from mininet.log import setLogLevel
from mininet.node import Docker
from time import sleep
if '__main__' == __name__:
setLogLevel('info')
net = Containernet(link=TCLink)
h1 = net.addHost('h1',ip="10.0.1.1/24")
r = net.addHost('r')
h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
net.addLink(h1, r, cls=TCLink, bw=10)
net.addLink(h2, r, cls=TCLink, bw=10)
net.addLink(h3, r, cls=TCLink, bw=10)
net.start()
h1,h2,h3,r=net.get('h1','h2','h3','r')
r.cmd("ifconfig r-eth0 0")
r.cmd("ifconfig r-eth1 0")
r.cmd("ifconfig r-eth2 0")
r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")
r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")
r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")
r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("ipvsadm -A -t 10.0.1.254:80 -s rr")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m")
h1.cmd("ip route add default via 10.0.1.254")
h2.cmd("ip route del default")
h2.cmd("ip route add default via 10.0.2.254")
h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")
h3.cmd("ip route del default")
h3.cmd("ip route add default via 10.0.3.254")
h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")
CLI(net)
net.stop()
```
這Script 本身沒有"Health Check" (需要額外Install software)
h2 -> using rate = 2%
h3 -> using rate = 2%
bandwidth = 10mbs
## SNAT
```
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("ipvsadm -A -t 10.0.1.254:80 -s rr")
```
## ipvsadm (LVS manager software
IPVS(ip virtual service) administration
[Reference](https://www.linuxcool.com/ipvsadm)
### 運作原理
```
#linux virtual service
r.cmd("ipvsadm -A -t 10.0.1.254:80 -s rr")
#ADD Real server Location
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m")
```
-A -> add a Virtual Service
-t -> tcp
-s -> scheduling Method, rr (Round-robin)
-a -> 定虚拟服务添加真实服务器
-r -> Real server
-m -> lvs -> Nat
## Pull Image File
```
docker pull smallko/php-apache-dev:v10
```
## Docker Options (Setting Computation Power)
Limiting server use Resource
- cpu_period
Default Value -> 1000000 (microseconds) -> 0.1 Seconds
- cpu_quota(配額)
在一個 period 下 去指定這個 Container 可以使用 CPU 多少 microseconds(微秒)
e.g. if quota= 250000 period = 1000000
Usage Rate = 250000/1000000 = 25 % (container Resource)
Default : 100% (如果不進行Setting)
## R1 ipvsadm
```
root@ubuntu:/home/user/server-test/lvs_haproxy# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.1.254:http rr
-> 10.0.2.2:http Masq 1 0 0
-> 10.0.3.3:http Masq 1 0 0
```
10.0.1.254 -> 跑 Round-Robin
H1 去Access Web Service
```
root@ubuntu:/home/user/server-test/lvs_haproxy# curl 10.0.1.254:80/a.htm
h2
root@ubuntu:/home/user/server-test/lvs_haproxy# curl 10.0.1.254:80/a.htm
h3
```
R1 這裡會記錄 Access 的紀錄
```
root@ubuntu:/home/user/server-test/lvs_haproxy# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.1.254:80 rr
-> 10.0.2.2:80 Masq 1 0 3
-> 10.0.3.3:80 Masq 1 0 3
```
InActConn
0-->3
0-->3
## 沒有Health Check Situation
Shut down H3 server(python -m )
```
root@ubuntu:/home/user# docker exec -it mn.h3 bash
root@h3:/# pkill python
```
H1 request service
```
root@ubuntu:/home/user/server-test/lvs_haproxy# curl 10.0.1.254:80/a.htm
h2
root@ubuntu:/home/user/server-test/lvs_haproxy# curl 10.0.1.254:80/a.htm
curl: (7) Failed to connect to 10.0.1.254 port 80: Connection refused
```
在第二次請求 下回得到 Connection Refuse
R1 這裡不會察覺到異常
```
root@ubuntu:/home/user/server-test/lvs_haproxy# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.1.254:80 rr
-> 10.0.2.2:80 Masq 1 0 2
-> 10.0.3.3:80 Masq 1 0 0
```
## script 2
Scheduling Methods wrr(weight Round Robin)
```
#!/usr/bin/env python
from mininet.net import Containernet
from mininet.cli import CLI
from mininet.link import Link,TCLink,Intf
from mininet.log import setLogLevel
from mininet.node import Docker
from time import sleep
if '__main__' == __name__:
setLogLevel('info')
net = Containernet(link=TCLink)
h1 = net.addHost('h1',ip="10.0.1.1/24")
r = net.addHost('r')
h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
net.addLink(h1, r, cls=TCLink, bw=10)
net.addLink(h2, r, cls=TCLink, bw=10)
net.addLink(h3, r, cls=TCLink, bw=10)
net.start()
h1,h2,h3,r=net.get('h1','h2','h3','r')
r.cmd("ifconfig r-eth0 0")
r.cmd("ifconfig r-eth1 0")
r.cmd("ifconfig r-eth2 0")
r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")
r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")
r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")
r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("ipvsadm -A -t 10.0.1.254:80 -s wrr")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m -w 1")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m -w 2")
h1.cmd("ip route add default via 10.0.1.254")
h2.cmd("ip route del default")
h2.cmd("ip route add default via 10.0.2.254")
h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")
h3.cmd("ip route del default")
h3.cmd("ip route add default via 10.0.3.254")
h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")
CLI(net)
net.stop()
```
延續上個script Ipvsadm 有做一些更動
使用 wrr algorithm
可以設定去 Weight 的 Value
```
r.cmd("ipvsadm -A -t 10.0.1.254:80 -s wrr")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.2.2:80 -m -w 1")
r.cmd("ipvsadm -a -t 10.0.1.254:80 -r 10.0.3.3:80 -m -w 2")
```
-w capacity of real server
# Lvs-DR(Direct Routing)
Topology
## DS (Directory Server)
處理 Client <-----> Server 之間的 inbound, outbound
DS 會需要2個 Interface
DIP -> (For 內網)
VIP -> (For 外網)
Request 經過DS 會修改
## Notice:
Rs 上需要修改kernal parameter 限制ARP 的Announce 跟 request, response
ARP Announcement: 用於update ARP cache
Dst -> ff:ff:ff:ff:ff:ff
## Script
```
#!/usr/bin/env python
from mininet.net import Containernet
from mininet.cli import CLI
from mininet.link import Link,TCLink,Intf
from mininet.log import setLogLevel
from mininet.node import Docker
from time import sleep
if '__main__' == __name__:
setLogLevel('info')
net = Containernet(link=TCLink)
h1 = net.addHost('h1',ip="10.0.1.1/24")
r = net.addHost('r')
br = net.addHost('br')
lb = net.addHost('lb', mac = '00:00:00:00:01:01', ip="10.0.2.1/24")
h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.2.3/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
net.addLink(h1, r, cls=TCLink, bw=10)
net.addLink(br, r, cls=TCLink, bw=10)
net.addLink(lb, br, cls=TCLink, bw=10)
net.addLink(h2, br, cls=TCLink, bw=10)
net.addLink(h3, br, cls=TCLink, bw=10)
net.start()
h1,h2,h3,lb,br,r=net.get('h1','h2','h3','lb','br','r')
r.cmd("ifconfig r-eth0 0")
r.cmd("ifconfig r-eth1 0")
r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")
r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")
r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")
br.cmd("ifconfig br-eth0 0")
br.cmd("ifconfig br-eth1 0")
br.cmd("ifconfig br-eth2 0")
br.cmd("ifconfig br-eth3 0")
br.cmd("brctl addbr br0")
br.cmd("brctl addif br0 br-eth0")
br.cmd("brctl addif br0 br-eth1")
br.cmd("brctl addif br0 br-eth2")
br.cmd("brctl addif br0 br-eth3")
br.cmd("ifconfig br0 up")
lb.cmd("ifconfig lb-eth0:0 10.0.2.100/24")
lb.cmd("ip route add default via 10.0.2.254")
lb.cmd("ipvsadm -A -t 10.0.2.100:80 -s rr")
lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.2 -g")
lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.3 -g")
h1.cmd("ip route add default via 10.0.1.254")
h2.cmd("ip route del default")
h2.cmd("ip route add default via 10.0.2.254")
h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")
h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")
h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")
h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")
h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")
h2.cmd("ifconfig lo:0 10.0.2.100 netmask 255.255.255.255 broadcast 10.0.2.100")
h2.cmd("route add -host 10.0.2.100 dev lo:0")
h3.cmd("ip route del default")
h3.cmd("ip route add default via 10.0.2.254")
h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")
h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")
h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")
h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")
h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")
h3.cmd("ifconfig lo:0 10.0.2.100 netmask 255.255.255.255 broadcast 10.0.2.100")
h3.cmd("route add -host 10.0.2.100 dev lo:0")
CLI(net)
net.stop()
```
## Node IP 設定
### DS (Director Server)
DS -> VIP Bind Ens33上
ens33
```
lb = net.addHost('lb', mac = '00:00:00:00:01:01', ip="10.0.2.1/24")
```
VIP -> 10.0.2.100
```
lb.cmd("ifconfig lb-eth0:0 10.0.2.100/24")
```
RS(Real server) 的 VIP(virtual IP) Bind 在 lo interface 上
## ipvsadm
```
#lb.cmd("ipvsadm -A -t 10.0.2.100:80 -s rr")
#lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.2 -g")
#lb.cmd("ipvsadm -a -t 10.0.2.100:80 -r 10.0.2.3 -g")
```
-g Flag -> RD routing
## arp_ignore 跟 arp_announce
[Reference](https://www.360blogs.top/linux-arp_ignore-arp_announce/)
arp_ignore参数的作用是控制系统在收到外部的arp请求时，是否要返回arp响应。
如果ARP Ignore =1 (只響應 DST 為本地的IP ARP) RS 就不會 replay 來自外部的ARP Request
當經過DS後 RS Response 給clinet 之後的request 不會再經過 DS
因為 client 會知道 RS的MAC
Real server H2
```
h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")
h2.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")
```
---
Real server H3
```
h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore")
h3.cmd("echo 1 > /proc/sys/net/ipv4/conf/h2-eth0/arp_ignore")
```
### arp_announce
arp_announce = 2
Real server H2
```
h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")
h2.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")
```
Real server H3
```
h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce")
h3.cmd("echo 2 > /proc/sys/net/ipv4/conf/h2-eth0/arp_announce")
```
## LB (DS)
```
root@ubuntu:/home/user/server-test/lvs_haproxy# ifconfig
lb-eth0 Link encap:Ethernet HWaddr 00:00:00:00:01:01
inet addr:10.0.2.1 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:21 errors:0 dropped:0 overruns:0 frame:0
TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1726 (1.7 KB) TX bytes:688 (688.0 B)
lb-eth0:0 Link encap:Ethernet HWaddr 00:00:00:00:01:01
inet addr:10.0.2.100 Bcast:10.0.2.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
```
## Test
H1 Request Web Service
```
containernet> h1 curl 10.0.2.100:80/a.htm
h2
containernet> h1 curl 10.0.2.100:80/a.htm
h3
```
沒有 HAProxcy
```
containernet> h1 curl 10.0.2.100:80/a.htm
curl: (7) Failed to connect to 10.0.2.100 port 80: Connection refused
containernet> h1 curl 10.0.2.100:80/a.htm
h2
```
# HaProxcy
Topology
Script
```
#!/usr/bin/env python
from mininet.net import Containernet
from mininet.cli import CLI
from mininet.link import Link,TCLink,Intf
from mininet.log import setLogLevel
from mininet.node import Docker
from time import sleep
from datetime import datetime
import time
if '__main__' == __name__:
setLogLevel('info')
net = Containernet(link=TCLink)
h1 = net.addHost('h1',ip="10.0.1.1/24")
r = net.addHost('r')
h2 = net.addDocker('h2', mac = '00:00:00:00:02:02', ip="10.0.2.2/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
h3 = net.addDocker('h3', mac = '00:00:00:00:03:03', ip="10.0.3.3/24", dimage="smallko/php-apache-dev:v10",cpu_period=50000, cpu_quota=1000)
Link(h1, r)
Link(r, h2)
Link(r, h3)
net.build()
h1,h2,h3,r,=net.get('h1','h2','h3','r')
r.cmd("ifconfig r-eth0 0")
r.cmd("ifconfig r-eth1 0")
r.cmd("ifconfig r-eth2 0")
r.cmd("echo 1 > /proc/sys/net/ipv4/ip_forward")
r.cmd("ip addr add 10.0.1.254/24 brd + dev r-eth0")
r.cmd("ip addr add 10.0.2.254/24 brd + dev r-eth1")
r.cmd("ip addr add 10.0.3.254/24 brd + dev r-eth2")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.2.0/24 -o r-eth0 -j MASQUERADE")
r.cmd("iptables -t nat -A POSTROUTING -s 10.0.3.0/24 -o r-eth0 -j MASQUERADE")
h1.cmd("ip route add default via 10.0.1.254")
h2.cmd("ip route del default")
h2.cmd("ip route add default via 10.0.2.254")
h2.cmd("cd /var/www/html; echo h2 > a.htm ; python -m SimpleHTTPServer 80 &")
h3.cmd("ip route del default")
h3.cmd("ip route add default via 10.0.3.254")
h3.cmd("cd /var/www/html; echo h3 > a.htm ; python -m SimpleHTTPServer 80 &")
CLI(net)
net.stop()
```
Configuration File
```
listen test
bind 10.0.1.254:80 #setting VIP
mode tcp
balance static-rr #scheduling method
balance roundrobin
server web1 10.0.2.2:80 check weight 1 check inter 1s #interval -> 1秒 check 1 次
server web2 10.0.3.3:80 check weight 1 check inter 1s
```
## R1 -> 運行 HA proxy
```
root@ubuntu:/home/user/server-test/lvs_haproxy# ls
haproxy.cfg test-haproxy.py test-lvs-nat-rr.py
lvs_haproxy.zip test-lvs-dr.py test-lvs-nat-wrr.py
root@ubuntu:/home/user/server-test/lvs_haproxy# /usr/sbin/haproxy -f ./haproxy.
cfg
[WARNING] 148/151933 (8335) : config : missing timeouts for proxy 'test'.
| While not properly invalid, you will certainly encounter various problems
| with such a configuration. To fix this, please ensure that all following
| timeouts are set to a non-zero value: 'client', 'connect', 'server'.
```
[](https://hackmd.io/_uploads/rky7mCbI2.png)
Shut down h3 web server
```
root@ubuntu:/home/user/server-test/lvs_haproxy# /usr/sbin/haproxy -f ./haproxy.
cfg
[WARNING] 148/151943 (8337) : config : missing timeouts for proxy 'test'.
| While not properly invalid, you will certainly encounter various problems
| with such a configuration. To fix this, please ensure that all following
| timeouts are set to a non-zero value: 'client', 'connect', 'server'.
[WARNING] 148/152246 (8337) : Server test/web2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue.
```
Warning Message
```
Server test/web2 is DOWN, reason: Layer4 connection problem, info: "Connection refused"
```
H3 會被HA proxy 跳過
```
containernet> h1 curl 10.0.1.254:80/a.htm
h2
containernet> h1 curl 10.0.1.254:80/a.htm
h2
containernet> h1 curl 10.0.1.254:80/a.htm
h2
containernet> h1 curl 10.0.1.254:80/a.htm
```
Sign in with Wallet
Connect another wallet