diff --git a/package.json b/package.json index 2589e561..beeb9c31 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,6 @@ "cidr-tools": "^3.0.5", "glob": "^7.2.3", "gulp": "^4.0.2", - "gulp-debug": "^4.0.0", "ipdb": "^0.3.3", "progress": "^2.0.3", "through2": "^4.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b4812df..553f7386 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,6 @@ importers: gulp: specifier: ^4.0.2 version: 4.0.2 - gulp-debug: - specifier: ^4.0.0 - version: 4.0.0(gulp@4.0.2) ipdb: specifier: ^0.3.3 version: 0.3.3 @@ -53,10 +50,6 @@ packages: resolution: {integrity: sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==} engines: {node: '>=0.10.0'} - ansi-styles@3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - ansi-wrap@0.1.0: resolution: {integrity: sha512-ZyznvL8k/FZeQHr2T6LzcJ/+vBApDnMNZvfVFy3At0knswWd6rJ3/0Hhmpu8oqa6C92npmozs890sX9Dl6q+Qw==} engines: {node: '>=0.10.0'} @@ -187,10 +180,6 @@ packages: resolution: {integrity: sha512-4nhGqUkc4BqbBBB4Q6zLuD7lzzrHYrjKGeYaEji/3tFR5VdJu9v+LilhGIVe8wxEJPPOeWo7eg8dwY13TZ1BNg==} engines: {node: '>=0.10.0'} - chalk@2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} @@ -235,12 +224,6 @@ packages: resolution: {integrity: sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==} engines: {node: '>=0.10.0'} - color-convert@1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - - color-name@1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} hasBin: true @@ -362,10 +345,6 @@ packages: es6-weak-map@2.0.3: resolution: {integrity: sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==} - escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - esniff@2.0.1: resolution: {integrity: sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==} engines: {node: '>=0.10'} @@ -471,9 +450,6 @@ packages: resolution: {integrity: sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==} engines: {node: '>= 0.4'} - get-own-enumerable-property-symbols@3.0.2: - resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} - get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -521,12 +497,6 @@ packages: engines: {node: '>= 0.10'} hasBin: true - gulp-debug@4.0.0: - resolution: {integrity: sha512-cn/GhMD2nVZCVxAl5vWao4/dcoZ8wUJ8w3oqTvQaGDmC1vT7swNOEbhQTWJp+/otKePT64aENcqAQXDcdj5H1g==} - engines: {node: '>=6'} - peerDependencies: - gulp: '>=4' - gulp@4.0.2: resolution: {integrity: sha512-dvEs27SCZt2ibF29xYgmnwwCYZxdxhQ/+LFWlbAW8y7jt68L/65402Lz3+CKy0Ov4rOs+NERmDq7YlZaDqUIfA==} engines: {node: '>= 0.10'} @@ -536,10 +506,6 @@ packages: resolution: {integrity: sha512-hm6N8nrm3Y08jXie48jsC55eCZz9mnb4OirAStEk2deqeyhXU3C1otDVh+ccttMuc1sBi6RX6ZJ720hs9RCvgw==} engines: {node: '>= 0.10'} - has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - has-property-descriptors@1.0.2: resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} @@ -615,10 +581,6 @@ packages: ipv6-normalize@1.0.1: resolution: {integrity: sha512-Bm6H79i01DjgGTCWjUuCjJ6QDo1HB96PT/xCYuyJUP9WFbVDrLSbG4EZCvOCun2rNswZb0c3e4Jt/ws795esHA==} - irregular-plurals@2.0.0: - resolution: {integrity: sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==} - engines: {node: '>=6'} - is-absolute@1.0.0: resolution: {integrity: sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==} engines: {node: '>=0.10.0'} @@ -697,10 +659,6 @@ packages: resolution: {integrity: sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==} engines: {node: '>=0.10.0'} - is-obj@1.0.1: - resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} - engines: {node: '>=0.10.0'} - is-plain-object@2.0.4: resolution: {integrity: sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==} engines: {node: '>=0.10.0'} @@ -709,10 +667,6 @@ packages: resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} engines: {node: '>=0.10.0'} - is-regexp@1.0.0: - resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} - engines: {node: '>=0.10.0'} - is-relative@1.0.0: resolution: {integrity: sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==} engines: {node: '>=0.10.0'} @@ -900,10 +854,6 @@ packages: ordered-read-streams@1.0.1: resolution: {integrity: sha512-Z87aSjx3r5c0ZB7bcJqIgIRX5bxR7A4aSzvIbaxd0oTkWBCOoKfuGHiKj60CHVUgg1Phm5yMZzBdt8XqRs73Mw==} - os-homedir@1.0.2: - resolution: {integrity: sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==} - engines: {node: '>=0.10.0'} - os-locale@1.4.0: resolution: {integrity: sha512-PRT7ZORmwu2MEFt4/fv3Q+mEfN4zetKxufQrkShY2oGvUms9r8otu5HfdyIFHkYXjO7laNsoVGmM2MANfuTA8g==} engines: {node: '>=0.10.0'} @@ -966,10 +916,6 @@ packages: resolution: {integrity: sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==} engines: {node: '>=0.10.0'} - plur@3.1.1: - resolution: {integrity: sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==} - engines: {node: '>=6'} - posix-character-classes@0.1.1: resolution: {integrity: sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==} engines: {node: '>=0.10.0'} @@ -1174,10 +1120,6 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - stringify-object@3.3.0: - resolution: {integrity: sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==} - engines: {node: '>=4'} - strip-ansi@3.0.1: resolution: {integrity: sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==} engines: {node: '>=0.10.0'} @@ -1186,10 +1128,6 @@ packages: resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} engines: {node: '>=0.10.0'} - supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - supports-preserve-symlinks-flag@1.0.0: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} @@ -1206,10 +1144,6 @@ packages: through2@4.0.2: resolution: {integrity: sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==} - tildify@1.2.0: - resolution: {integrity: sha512-Y9q1GaV/BO65Z9Yf4NOGMuwt3SGdptkZBnaaKfTQakrDyCLiuO1Kc5wxW4xLdsjzunRtqtOdhekiUFmZbklwYQ==} - engines: {node: '>=0.10.0'} - time-stamp@1.1.0: resolution: {integrity: sha512-gLCeArryy2yNTRzTGKbZbloctj64jkZ57hj5zdraXue6aFgd6PmvVtEyiUU+hvU0v7q08oVv8r8ev0tRo6bvgw==} engines: {node: '>=0.10.0'} @@ -1346,10 +1280,6 @@ snapshots: ansi-regex@2.1.1: {} - ansi-styles@3.2.1: - dependencies: - color-convert: 1.9.3 - ansi-wrap@0.1.0: {} anymatch@2.0.0: @@ -1503,12 +1433,6 @@ snapshots: camelcase@3.0.0: {} - chalk@2.4.2: - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - chokidar@2.1.8: dependencies: anymatch: 2.0.0 @@ -1579,12 +1503,6 @@ snapshots: map-visit: 1.0.0 object-visit: 1.0.1 - color-convert@1.9.3: - dependencies: - color-name: 1.1.3 - - color-name@1.1.3: {} - color-support@1.1.3: {} component-emitter@1.3.1: {} @@ -1714,8 +1632,6 @@ snapshots: es6-iterator: 2.0.3 es6-symbol: 3.1.4 - escape-string-regexp@1.0.5: {} - esniff@2.0.1: dependencies: d: 1.0.2 @@ -1869,8 +1785,6 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 - get-own-enumerable-property-symbols@3.0.2: {} - get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -1962,16 +1876,6 @@ snapshots: transitivePeerDependencies: - supports-color - gulp-debug@4.0.0(gulp@4.0.2): - dependencies: - chalk: 2.4.2 - fancy-log: 1.3.3 - gulp: 4.0.2 - plur: 3.1.1 - stringify-object: 3.3.0 - through2: 2.0.5 - tildify: 1.2.0 - gulp@4.0.2: dependencies: glob-watcher: 5.0.5 @@ -1985,8 +1889,6 @@ snapshots: dependencies: glogg: 1.0.2 - has-flag@3.0.0: {} - has-property-descriptors@1.0.2: dependencies: es-define-property: 1.0.1 @@ -2056,8 +1958,6 @@ snapshots: ipv6-normalize@1.0.1: {} - irregular-plurals@2.0.0: {} - is-absolute@1.0.0: dependencies: is-relative: 1.0.0 @@ -2129,16 +2029,12 @@ snapshots: is-number@4.0.0: {} - is-obj@1.0.1: {} - is-plain-object@2.0.4: dependencies: isobject: 3.0.1 is-plain-object@5.0.0: {} - is-regexp@1.0.0: {} - is-relative@1.0.0: dependencies: is-unc-path: 1.0.0 @@ -2361,8 +2257,6 @@ snapshots: dependencies: readable-stream: 2.3.8 - os-homedir@1.0.2: {} - os-locale@1.4.0: dependencies: lcid: 1.0.0 @@ -2413,10 +2307,6 @@ snapshots: pinkie@2.0.4: {} - plur@3.1.1: - dependencies: - irregular-plurals: 2.0.0 - posix-character-classes@0.1.1: {} pretty-hrtime@1.0.3: {} @@ -2644,12 +2534,6 @@ snapshots: dependencies: safe-buffer: 5.2.1 - stringify-object@3.3.0: - dependencies: - get-own-enumerable-property-symbols: 3.0.2 - is-obj: 1.0.1 - is-regexp: 1.0.0 - strip-ansi@3.0.1: dependencies: ansi-regex: 2.1.1 @@ -2658,10 +2542,6 @@ snapshots: dependencies: is-utf8: 0.2.1 - supports-color@5.5.0: - dependencies: - has-flag: 3.0.0 - supports-preserve-symlinks-flag@1.0.0: {} sver-compat@1.5.0: @@ -2683,10 +2563,6 @@ snapshots: dependencies: readable-stream: 3.6.2 - tildify@1.2.0: - dependencies: - os-homedir: 1.0.2 - time-stamp@1.1.0: {} to-absolute-glob@2.0.2: diff --git a/src/gulpfile.js b/src/gulpfile.js index 916b71ee..dca1a837 100644 --- a/src/gulpfile.js +++ b/src/gulpfile.js @@ -5,6 +5,7 @@ const plugin_city = require('./plugins/city') const plugin_cncity = require('./plugins/cncity') const plugin_china = require('./plugins/china') const plugin_cidrmerge = require('./plugins/cidrmerge') +const plugin_isp = require('./plugins/isp') const database = '/tmp/openipdb.ipdb' @@ -35,6 +36,15 @@ const cncity = () => { .pipe(dest('data/cncity')) } +const isp = () => { + return src(database) + .pipe(through2.obj(function(file, _, cb) { + return plugin_isp(this, file, cb) + })) + .pipe(through2.obj(plugin_cidrmerge)) + .pipe(dest('data/isp')) +} + const china = () => { return src('data/country/CN.txt') .pipe(through2.obj(function(file, _, cb) { @@ -47,4 +57,5 @@ exports.country = country exports.city = city exports.cncity = cncity exports.china = china -exports.build = series(country, city, cncity, china) +exports.isp = isp +exports.build = series(country, city, cncity, isp, china) diff --git a/src/plugins/isp.js b/src/plugins/isp.js new file mode 100644 index 00000000..c9097cb4 --- /dev/null +++ b/src/plugins/isp.js @@ -0,0 +1,59 @@ +const vinyl = require('vinyl') +const IPDB = require('ipdb') +const ipdb_range = require('@ipdb/range') +const ProgressBar = require('progress') + +const ISP_MAP = { + // 中国电信 + 'chinatelecom.com.cn': 'chinatelecom', + // 中国联通 + 'chinaunicom.com': 'chinaunicom', + // 中国移动 + 'chinamobile.com': 'chinamobile', + // 中国教育网 + 'cernet.edu.cn': 'cernet', +} + +const plugin = (through2, file, cb) => { + + console.log('Parse ipdb') + + const ipdb = new IPDB(file.contents, { + patches: [ipdb_range] + }) + + let bar = new ProgressBar(':bar :current/:total', { total: ipdb.meta.node_count }) + + let result = [] + let ip = '0.0.0.0' + while (true) { + const info = ipdb.find(ip).data + const isp = info.isp_domain || '' + if (isp.length > 0 && ISP_MAP[isp]) { + const ispName = ISP_MAP[isp] + if (!result[ispName]) { + result[ispName] = [] + } + result[ispName].push(`${info.range.from}/${info.bitmask}`) + } + bar.tick() + ip = info.range.next + if (ip === '0.0.0.0') break + } + + console.log() + + for (let [isp, cidrs] of Object.entries(result)) { + let temp = new vinyl({ + cwd: '/', + base: '/', + path: `/${isp}.txt`, + contents: new Buffer.from(cidrs.join('\n')) + }) + through2.push(temp) + } + + cb() +} + +module.exports = plugin