From bf6d9cf89778336375269492544f5b51e25ceb11 Mon Sep 17 00:00:00 2001
From: luwenpeng <luwenpeng@zdjizhi.com>
Date: Thu, 18 Mar 2021 10:02:34 +0800
Subject: [PATCH] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=20tfe=20=E5=AE=B9=E5=99=A8?=
 =?UTF-8?q?=E7=9A=84=E8=BF=90=E8=A1=8C=E7=8E=AF=E5=A2=83=20=091.=E9=80=8F?=
 =?UTF-8?q?=E4=BC=A0=20TAP=20=E7=BD=91=E5=8D=A1=E5=88=B0=20tfe=20=E7=9A=84?=
 =?UTF-8?q?=E5=AE=B9=E5=99=A8=E9=87=8C=20=092.=E5=9C=A8=E5=AE=BF=E4=B8=BB?=
 =?UTF-8?q?=E6=9C=BA=E8=AE=BE=E7=BD=AE=20sysctl=20=E5=8F=82=E6=95=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 dockerfile/tfe/supervisord.conf | 14 +++---
 init_tfe_env.sh                 | 79 +++++++++++++++++++++++++++++++++
 tfe_sysctl.conf                 | 32 +++++++++++++
 3 files changed, 118 insertions(+), 7 deletions(-)
 create mode 100644 init_tfe_env.sh
 create mode 100644 tfe_sysctl.conf

diff --git a/dockerfile/tfe/supervisord.conf b/dockerfile/tfe/supervisord.conf
index 3cd3cc9..adc1a56 100644
--- a/dockerfile/tfe/supervisord.conf
+++ b/dockerfile/tfe/supervisord.conf
@@ -169,14 +169,14 @@ serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL  for a unix socket
 ;[include]
 ;files = relative/directory/*.ini
 
-[program:tfe-env]
-user=root
-Command=bash -c "/opt/tsg/tfe/tfe-env.sh start"
-directory=/opt/tsg/tfe/
+;[program:tfe-env]
+;user=root
+;Command=bash -c "/opt/tsg/tfe/tfe-env.sh start"
+;directory=/opt/tsg/tfe/
 
-startsecs=0
-startretries=0
-autorestart=false
+;startsecs=0
+;startretries=0
+;autorestart=false
 
 [program:tfe]
 user=root
diff --git a/init_tfe_env.sh b/init_tfe_env.sh
new file mode 100644
index 0000000..0c29f78
--- /dev/null
+++ b/init_tfe_env.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+NIC=tap0
+PEER_IP=172.16.2.1
+LOCAL_IP=172.16.2.2
+LOCAL_MAC=fe:65:b7:00:00:01
+container=tfe-container-v4.4
+
+exposedockernetns ()
+{
+	pid=`docker inspect -f '{{.State.Pid}}' $1`
+	ln -s /proc/$pid/ns/net /var/run/netns/$1
+	echo "netns of ${1} exposed as /var/run/netns/${1}"
+}
+
+dockerrmf ()
+{
+	docker kill `docker ps --no-trunc -aq`
+	docker rm `docker ps --no-trunc -aq`
+}
+
+cp tfe_sysctl.conf /etc/sysctl.d/
+sysctl -p /etc/sysctl.d/tfe_sysctl.conf
+
+#dockerrmf
+#ip tuntap del dev ${NIC} mode tap
+#ip tuntap add dev ${NIC} mode tap one_queue
+ip netns list
+
+###############################################################################
+# 注意：需要停止 tfe-env 脚本
+# cd /home/lwp/tsg_container
+# docker-compose up tfe
+###############################################################################
+
+rm -Rf /var/run/netns/*
+mkdir -p /var/run/netns
+exposedockernetns ${container}
+ip link set ${NIC} netns ${container}
+
+ip netns exec ${container} ip link set ${NIC} address ${LOCAL_MAC}
+ip netns exec ${container} ip link set ${NIC} up
+
+# IPv4 策略路由
+ip netns exec ${container} ip addr add ${LOCAL_IP}/24 dev ${NIC}
+
+ip netns exec ${container} ip rule add iif ${NIC} tab 100
+ip netns exec ${container} ip route add local default dev lo table 100
+
+ip netns exec ${container} ip rule add fwmark 0x65 lookup 101
+ip netns exec ${container} ip route add default dev ${NIC} via ${PEER_IP} table 101
+
+# IPv6 策略路由
+ip netns exec ${container} ip addr add fd00::02/64 dev ${NIC}
+
+ip netns exec ${container} ip -6 route add default via fd00::01
+
+ip netns exec ${container} ip -6 rule add iif ${NIC} tab 102
+ip netns exec ${container} ip -6 route add local default dev lo table 102
+
+# iptables
+ip netns exec ${container} iptables -A INPUT -i ${NIC} -m bpf \
+	--bytecode '14,48 0 0 0,84 0 0 240,21 0 10 64,48 0 0 9,21 0 8 6,40 0 0 6,69 6 0 8191,177 0 0 0,80 0 0 20,21 0 3 88,80 0 0 21,21 0 1 4,6 0 0 65535,6 0 0 0' \
+	-j NFQUEUE --queue-num 1
+
+echo "================ run 'ip addr list'  in container ================"
+ip netns exec ${container} ip addr list
+
+echo "================ run 'ip rule list'  in container ================"
+ip netns exec ${container} ip rule list
+
+echo "================ run 'ip route list' in container ================"
+ip netns exec ${container} ip route list
+
+echo "================ run 'iptables'      in container ================"
+ip netns exec ${container} iptables -L
+
+echo "================ run 'ping'          in container ================"
+ip netns exec ${container} ping -c10 ${PEER_IP}
diff --git a/tfe_sysctl.conf b/tfe_sysctl.conf
new file mode 100644
index 0000000..f8ec209
--- /dev/null
+++ b/tfe_sysctl.conf
@@ -0,0 +1,32 @@
+# read/write buffer
+net.core.rmem_default = 256960
+net.core.rmem_max = 256960
+net.core.wmem_default = 256960
+net.core.wmem_max = 256960
+
+# disable rpfilter
+net.ipv4.conf.all.rp_filter=0
+net.ipv4.conf.default.rp_filter=0
+
+# fs
+fs.file-max=1048576
+net.core.netdev_max_backlog=1000000
+net.core.somaxconn=65535
+
+# tcp options about TIME_WAIT
+net.ipv4.tcp_fin_timeout=10
+net.ipv4.tcp_tw_reuse=1
+net.ipv4.tcp_max_tw_buckets=4096
+net.ipv4.tcp_max_orphans=131072
+net.ipv4.tcp_max_syn_backlog=131072
+
+# bbr
+#net.ipv4.tcp_congestion_control=bbr
+
+# tcp feature
+net.ipv4.tcp_ecn=0
+net.ipv4.tcp_sack=1
+net.ipv4.tcp_timestamps=1
+
+# disable tcp windows scaling for kernel bugs
+net.ipv4.tcp_window_scaling=0