Merge branch 'master' into haproxy

ansible/configure.yaml

@@ -222,10 +222,8 @@
       docker_compose:
         project_src: '{{ansible_user_dir}}/nextgen-network/services'
         restarted: true
-        services:
-          - frpc-{{item.name}}
-      with_items: '{{connections}}'
-      when: 'item.protocol == "wgfrp" and item.frpType == "frpc" and not noBird and not item.noUpdate'
+        services: '{{frpcRestarts}}'
+      when: frpcRestarts
     - name: restart_ocserv
       docker_compose:
         project_src: '{{ansible_user_dir}}/nextgen-network/services'
@@ -237,10 +235,8 @@
       docker_compose:
         project_src: '{{ansible_user_dir}}/nextgen-network/services'
         restarted: true
-        services:
-          - openconnect-{{item.name}}
-      with_items: '{{connections}}'
-      when: 'item.protocol == "oc" and item.ocType == "client" and not noBird and not item.noUpdate'
+        services: '{{ocRestarts}}'
+      when: ocRestarts
     #- name: restart_bird_systemd
     #  become: true
     #  systemd:

ansible/scripts/switch-rules-up.sh.j2

 #!/bin/bash
+## reloaded at 6.24
 source {{ansible_user_dir}}/nextgen-network/scripts/utility.sh
 
 ip rule add pref 300 fwmark 999 table 999

config-with-proxy.sh

 #!/bin/bash
-LADDER="water"
-ssh root@10.198.1.1 -p 55322 ipset add u_${LADDER}_chnroute 10.198.1.57
-ssh nanahira@10.198.0.6 sudo ipset add u_${LADDER}_chnroute 10.198.1.57
+LADDER="jue"
+ssh root@10.198.1.1 -p 55322 ipset add u_${LADDER}_oversea 10.198.1.57
+#ssh nanahira@10.198.0.6 sudo ipset add u_${LADDER}_oversea 10.198.1.57
 ./config.sh
-ssh root@10.198.1.1 -p 55322 ipset del u_${LADDER}_chnroute 10.198.1.57
-ssh nanahira@10.198.0.6 sudo ipset del u_${LADDER}_chnroute 10.198.1.57
+ssh root@10.198.1.1 -p 55322 ipset del u_${LADDER}_oversea 10.198.1.57
+#ssh nanahira@10.198.0.6 sudo ipset del u_${LADDER}_oversea 10.198.1.57

config.sh

@@ -9,3 +9,7 @@ do
     echo "https://docs.google.com/spreadsheets/d/${doc}/export?exportFormat=csv&gid=${sheet}"
 done ) | wget -4 --content-disposition -i -
 cd ..
+
+cd lists
+./run.sh
+cd ..

package-lock.json

@@ -17,6 +17,11 @@
       "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz",
       "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ=="
     },
+    "@types/mustache": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.1.1.tgz",
+      "integrity": "sha512-Sm0NWeLhS2QL7NNGsXvO+Fgp7e3JLHCO6RS3RCnfjAnkw6Y1bsji/AGfISdQZDIR/AeOyzkrxRk9jBkl55zdJw=="
+    },
     "@types/node": {
       "version": "13.7.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz",
@@ -61,6 +66,11 @@
       "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==",
       "dev": true
     },
+    "mustache": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
+      "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ=="
+    },
     "source-map": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

package.json

@@ -10,9 +10,11 @@
   "dependencies": {
     "@types/ip": "^1.1.0",
     "@types/lodash": "^4.14.149",
+    "@types/mustache": "^4.1.1",
     "csv-parse": "^4.8.5",
     "ip": "^1.1.5",
     "lodash": "^4.17.15",
+    "mustache": "^4.2.0",
     "yaml": "^1.10.0"
   },
   "devDependencies": {

remove-interface.sh

@@ -4,6 +4,6 @@ COMMAND="flush interface ${INTERFACE}\nquit"
 COMMAND_BASE64=$(echo -e "$COMMAND" | base64 -w 0)
 
 cd ansible
-ansible wg -m shell -a "echo $COMMAND_BASE64 | base64 --decode | timeout 1 nc ::1 33123"
+ansible wg -m shell -b -a "wg-quick down ${INTERFACE} ; echo $COMMAND_BASE64 | base64 --decode | timeout 1 nc ::1 33123"
 ansible wg -m systemd -b -a "name=wg-quick@${INTERFACE} state=stopped enabled=false"
 cd ..

src/inventory.ts

@@ -17,7 +17,7 @@ async function generateOcpasswdLine(username: string, password: string) {
   return res;
 }
 
-interface GatewayGroup {
+interface GatewayGroup extends Record<string, any> {
   id: number;
   name: string;
   locationPrefix: string;
@@ -27,15 +27,17 @@ interface GatewayGroup {
   destMark: number;
 }
 
+type CommonEntry = Record<string, any>;
+
 class InventoryBuilder {
-  hosts: { [key: string]: any };
-  gateways: any;
+  hosts: { [key: string]: CommonEntry };
+  gateways: Record<string, CommonEntry>;
   gatewayGroups: GatewayGroup[];
   connections: string[];
-  routeLists: any;
+  routeLists: Record<string, string[]>;
   resolveCache: Map<string, string>;
   resolver: dns.Resolver;
-  vars: any;
+  vars: CommonEntry;
   linksOnly: string[];
   linksLimit: string[];
 
@@ -45,6 +47,14 @@ class InventoryBuilder {
     this.resolver.setServers(process.env.DNS ? [process.env.DNS] : ['114.114.114.114', '223.5.5.5']);
   }
 
+  getDockerImageTag(host: any) {
+    if (host.arch && host.arch.length) {
+      return `:master-${host.arch}`;
+    } else {
+      return '';
+    }
+  }
+
   async resolveDomain(domain: string, ipv6: boolean) {
     if (!domain || domain.match(/(\d{1,3}\.){3}\d{1,3}/)) {
       return domain;
@@ -70,7 +80,7 @@ class InventoryBuilder {
     return resolvedIP;
   }
 
-  async load(sheetName: string) {
+  async load(sheetName: string): Promise<Record<string, any>[]> {
     const data = await fs.promises.readFile(path.join('data', `内网互联计划 - ${sheetName}.csv`));
     // @ts-ignore
     return (await util.promisify(parse)(data, { columns: true, cast: true })).filter(h => h.id);
@@ -95,7 +105,7 @@ class InventoryBuilder {
   async main() {
     this.hosts = _.keyBy(await this.load('nextgen2'), 'name');
     this.gateways = _.mapValues(_.groupBy(await this.loadGateways(), 'router'), g => _.keyBy(g, 'isp'));
-    this.gatewayGroups = await this.load('gateway groups');
+    this.gatewayGroups = await this.load('gateway groups') as GatewayGroup[];
     //console.log(this.gateways);
     this.connections = _.intersection(Object.keys(this.hosts), Object.keys(_.find(this.hosts)));
 
@@ -222,7 +232,7 @@ class InventoryBuilder {
       services: {
         'gateways-monitor': {
           restart: 'always',
-          image: 'git-registry.mycard.moe/railgun/gateways-monitor',
+          image: `git-registry.mycard.moe/railgun/gateways-monitor${this.getDockerImageTag(host)}`,
           network_mode: 'host',
           cap_add: ['NET_ADMIN'],
           volumes: ['./route-plans:/usr/src/app/route-plans:ro'],
@@ -233,13 +243,15 @@ class InventoryBuilder {
     if (!host.sysBird) {
       host.dockerServices.services.babeld = {
         restart: 'always',
-        image: 'git-registry.mycard.moe/railgun/babeld',
+        image: `git-registry.mycard.moe/railgun/babeld${this.getDockerImageTag(host)}`,
         network_mode: 'host',
         //cap_add: ['NET_ADMIN'],
         privileged: true,
         volumes: ['./babeld.conf:/etc/babeld.conf:ro']
       };
     }
+    host.frpcRestarts = [];
+    host.ocRestarts = [];
     host.frpsNeeded = false;
     const null_connection = '10000,null';
     const lanInterfaces = host.lanInterfaces;
@@ -294,6 +306,8 @@ class InventoryBuilder {
       key: host.wgPrivateKey,
       frpsNeeded: host.frpsNeeded,
       frpsPort: host.frpsPort,
+      ocRestarts: host.ocRestarts.length ? host.ocRestarts : false,
+      frpcRestarts: host.frpcRestarts.length ? host.frpcRestarts : false,
       ocservNeeded: host.ocservNeeded || false,
       ocservPort: host.ocservPort,
       ocservCert: host.ocservCert || null,
@@ -355,6 +369,8 @@ class InventoryBuilder {
     const frpType = protocol === 'wgfrp' ? (this.gatewayCompare(localGateway, remoteGateway) ? 'frps' : 'frpc') : undefined;
     const ocType = protocol === 'oc' ? (this.gatewayCompareOcserv(local, remote, localGateway, remoteGateway) ? 'server' : 'client') : undefined;
 
+    const noUpdate = this.linksOnly && !(this.linksOnly.includes(remote.name) || this.linksOnly.includes(local.name)) || this.linksLimit && !(this.linksLimit.includes(remote.name) && this.linksLimit.includes(local.name));
+
     if (frpType === 'frps' && !local.dockerServices.services.frps) {
       local.frpsNeeded = true;
       local.dockerServices.services.frps = {
@@ -367,13 +383,17 @@ class InventoryBuilder {
     }
 
     if (frpType === 'frpc') {
-      local.dockerServices.services[`frpc-${name}`] = {
+      const containerName = `frpc-${name}`;
+      local.dockerServices.services[containerName] = {
         restart: 'always',
         image: 'fatedier/frpc:v0.34.2',
         network_mode: 'host',
         command: '-c /frpc.ini',
         volumes: [`./frpc-${name}.ini:/frpc.ini:ro`]
       };
+      if (!noUpdate) {
+        local.frpcRestarts.push(containerName);
+      }
     }
 
     if (ocType === 'server') {
@@ -383,7 +403,7 @@ class InventoryBuilder {
         local.ocMetric = metric;
         local.dockerServices.services.ocserv = {
           restart: 'always',
-          image: 'git-registry.mycard.moe/nanahira/docker-ocserv',
+          image: `git-registry.mycard.moe/nanahira/docker-ocserv${this.getDockerImageTag(local)}`,
           network_mode: 'host',
           command: 'ocserv -f -d 1',
           cap_add: ['NET_ADMIN'],
@@ -411,9 +431,10 @@ class InventoryBuilder {
       if (params.p === "tcp") {
         startupCommand += ' --no-dtls';
       }
-      local.dockerServices.services[`openconnect-${name}`] = {
+      const containerName = `openconnect-${name}`;
+      local.dockerServices.services[containerName] = {
         restart: 'always',
-        image: 'git-registry.mycard.moe/railgun/openconnect',
+        image: `git-registry.mycard.moe/railgun/openconnect${this.getDockerImageTag(local)}`,
         network_mode: 'host',
         command: ['bash', '-c', startupCommand],
         cap_add: ['NET_ADMIN'],
@@ -426,6 +447,9 @@ class InventoryBuilder {
           '$HOME/nextgen-network/scripts:$HOME/nextgen-network/scripts:ro'
         ]
       };
+      if (!noUpdate) {
+        local.ocRestarts.push(containerName);
+      }
     }
 
     //console.log(local.name, name, mtu);
@@ -434,8 +458,6 @@ class InventoryBuilder {
       console.log(`${local.name} GW ${localGateway.isp} ${inbound ? '<' : '='}=${(frpType === 'frps' || ocType === 'server') ? 's' : '='}=[${protocol}]=${(frpType === 'frpc' || ocType === 'client') ? 's' : '='}=> ${remote.name} GW ${remoteGateway.isp}`);
     }
 
-    const noUpdate = this.linksOnly && !(this.linksOnly.includes(remote.name) || this.linksOnly.includes(local.name)) || this.linksLimit && !(this.linksLimit.includes(remote.name) && this.linksLimit.includes(local.name));
-
     return {
       name,
       metric,

update.sh

@@ -6,9 +6,9 @@ set -e
 
 #read -p '确认无误后按回车继续'
 
-cd lists
-./run.sh
-cd ..
+#cd lists
+#./run.sh
+#cd ..
 
 mkdir -p result
 npm run build