调整构建逻辑,移除无效文件

This commit is contained in:
metowolf
2025-02-06 17:10:17 +08:00
parent 8896a97f50
commit 488bc4e3bb
12 changed files with 3374 additions and 15 deletions

View File

@@ -8,16 +8,11 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout Builder
uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
path: build
ref: build
version: 9
- name: Checkout Data
uses: actions/checkout@v4
with:
path: data
ref: master
- name: Set up S3cmd cli tool
uses: s3-actions/s3cmd@v1.9.0
with:
@@ -26,22 +21,21 @@ jobs:
access_key: ${{ secrets.S3_ACCESS_KEY }}
secret_key: ${{ secrets.S3_SECRET_KEY }}
account_id: ${{ secrets.S3_ACCOUNT_ID }}
- name: Set Timezone
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: "Asia/Shanghai"
- name: Parse
run: |
s3cmd get s3://openipdb/openipdb.ipdb /tmp/openipdb.ipdb
ls -al /tmp/openipdb.ipdb
cd build
yarn install
yarn build
rm -r ./data
pnpm install
pnpm run build
- name: Push
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
cd ./data
cp -r ../build/data/cncity/* ./data/cncity/
cp -r ../build/data/country/* ./data/country/
cp -r ../build/data/special/* ./data/special/
if [ -z "$(git status --porcelain)" ]; then
echo "No changes to the output on this push; exiting."
exit 0

3
.gitignore vendored
View File

@@ -1,3 +1,6 @@
# 构建工具
node_modules/
# 排除台湾省区划细分
!data/cncity/710000.txt
data/cncity/7*

24
package.json Normal file
View File

@@ -0,0 +1,24 @@
{
"name": "iplist",
"version": "1.0.0",
"scripts": {
"build": "npm run gulp build",
"gulp": "npx --max-old-space-size=8092 gulp --gulpfile src/gulpfile.js --cwd ./",
"test": "npm run gulp test"
},
"repository": "git@github.com:metowolf/iplist.git",
"author": "metowolf <i@i-meto.com>",
"license": "MIT",
"private": true,
"dependencies": {
"@ipdb/range": "^0.1.1",
"cidr-tools": "^3.0.5",
"glob": "^7.2.3",
"gulp": "^4.0.2",
"gulp-debug": "^4.0.0",
"ipdb": "^0.3.3",
"progress": "^2.0.3",
"through2": "^4.0.2",
"yaml": "^1.10.2"
}
}

2514
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

50
src/gulpfile.js Normal file
View File

@@ -0,0 +1,50 @@
const { series, src, dest } = require('gulp')
const through2 = require('through2')
const plugin_country = require('./plugins/country')
const plugin_city = require('./plugins/city')
const plugin_cncity = require('./plugins/cncity')
const plugin_china = require('./plugins/china')
const plugin_cidrmerge = require('./plugins/cidrmerge')
const database = '/tmp/openipdb.ipdb'
const country = () => {
return src(database)
.pipe(through2.obj(function(file, _, cb) {
return plugin_country(this, file, cb)
}))
.pipe(through2.obj(plugin_cidrmerge))
.pipe(dest('data/country'))
}
const city = () => {
return src(database)
.pipe(through2.obj(function(file, _, cb) {
return plugin_city(this, file, cb)
}))
.pipe(through2.obj(plugin_cidrmerge))
.pipe(dest('data/country'))
}
const cncity = () => {
return src(database)
.pipe(through2.obj(function(file, _, cb) {
return plugin_cncity(this, file, cb)
}))
.pipe(through2.obj(plugin_cidrmerge))
.pipe(dest('data/cncity'))
}
const china = () => {
return src('data/country/CN.txt')
.pipe(through2.obj(function(file, _, cb) {
return plugin_china(this, file, cb)
}))
.pipe(dest('data/special'))
}
exports.country = country
exports.city = city
exports.cncity = cncity
exports.china = china
exports.build = series(country, city, cncity, china)

509
src/plugins/cac/data.js Normal file
View File

@@ -0,0 +1,509 @@
module.exports = {
'中国': {
'中国': 100000,
},
'北京': {
'北京': 110000,
'东城区': 110101,
'西城区': 110102,
'朝阳区': 110105,
'丰台区': 110106,
'石景山区': 110107,
},
'天津': {
'天津': 120000,
'和平区': 120101,
'河东区': 120102,
'河西区': 120103,
'南开区': 120104,
'河北区': 120105,
},
'河北': {
'河北': 130000,
'石家庄': 130100,
'长安区': 130102,
'桥西区': 130104,
'新华区': 130105,
'井陉矿区': 130107,
'裕华区': 130108,
'唐山': 130200,
'秦皇岛': 130300,
'邯郸': 130400,
'邢台': 130500,
'保定': 130600,
'张家口': 130700,
'承德': 130800,
'沧州': 130900,
'廊坊': 131000,
'衡水': 131100,
},
'山西': {
'山西': 140000,
'太原': 140100,
'大同': 140200,
'阳泉': 140300,
'长治': 140400,
'晋城': 140500,
'朔州': 140600,
'晋中': 140700,
'运城': 140800,
'忻州': 140900,
'临汾': 141000,
'吕梁': 141100,
},
'内蒙古': {
'内蒙古': 150000,
'呼和浩特': 150100,
'包头': 150200,
'乌海': 150300,
'赤峰': 150400,
'通辽': 150500,
'鄂尔多斯': 150600,
'呼伦贝尔': 150700,
'巴彦淖尔': 150800,
'乌兰察布': 150900,
'兴安盟': 152200,
'锡林郭勒盟': 152500,
'阿拉善盟': 152900,
},
'辽宁': {
'辽宁': 210000,
'沈阳': 210100,
'大连': 210200,
'鞍山': 210300,
'抚顺': 210400,
'本溪': 210500,
'丹东': 210600,
'锦州': 210700,
'营口': 210800,
'阜新': 210900,
'辽阳': 211000,
'盘锦': 211100,
'铁岭': 211200,
'朝阳': 211300,
'葫芦岛': 211400,
},
'吉林': {
'吉林': 220000,
'长春': 220100,
'吉林市': 220200,
'四平': 220300,
'辽源': 220400,
'通化': 220500,
'白山': 220600,
'松原': 220700,
'白城': 220800,
'延边朝鲜族自治州': 222400,
'延边州': 222400,
},
'黑龙江': {
'黑龙江': 230000,
'哈尔滨': 230100,
'齐齐哈尔': 230200,
'鸡西': 230300,
'鹤岗': 230400,
'双鸭山': 230500,
'大庆': 230600,
'伊春': 230700,
'佳木斯': 230800,
'七台河': 230900,
'牡丹江': 231000,
'黑河': 231100,
'绥化': 231200,
'大兴安岭地区': 232700,
},
'上海': {
'上海': 310000,
},
'江苏': {
'江苏': 320000,
'南京': 320100,
'无锡': 320200,
'徐州': 320300,
'常州': 320400,
'苏州': 320500,
'南通': 320600,
'连云港': 320700,
'淮安': 320800,
'盐城': 320900,
'扬州': 321000,
'镇江': 321100,
'泰州': 321200,
'宿迁': 321300,
},
'浙江': {
'浙江': 330000,
'杭州': 330100,
'宁波': 330200,
'温州': 330300,
'嘉兴': 330400,
'湖州': 330500,
'绍兴': 330600,
'金华': 330700,
'衢州': 330800,
'舟山': 330900,
'台州': 331000,
'丽水': 331100,
},
'安徽': {
'安徽': 340000,
'合肥': 340100,
'芜湖': 340200,
'蚌埠': 340300,
'淮南': 340400,
'马鞍山': 340500,
'淮北': 340600,
'铜陵': 340700,
'安庆': 340800,
'黄山': 341000,
'滁州': 341100,
'阜阳': 341200,
'宿州': 341300,
'六安': 341500,
'亳州': 341600,
'池州': 341700,
'宣城': 341800,
},
'福建': {
'福建': 350000,
'福州': 350100,
'厦门': 350200,
'莆田': 350300,
'三明': 350400,
'泉州': 350500,
'漳州': 350600,
'南平': 350700,
'龙岩': 350800,
'宁德': 350900,
},
'江西': {
'江西': 360000,
'南昌': 360100,
'景德镇': 360200,
'萍乡': 360300,
'九江': 360400,
'新余': 360500,
'鹰潭': 360600,
'赣州': 360700,
'吉安': 360800,
'宜春': 360900,
'抚州': 361000,
'上饶': 361100,
},
'山东': {
'山东': 370000,
'济南': 370100,
'青岛': 370200,
'淄博': 370300,
'枣庄': 370400,
'东营': 370500,
'烟台': 370600,
'潍坊': 370700,
'济宁': 370800,
'泰安': 370900,
'威海': 371000,
'日照': 371100,
'临沂': 371300,
'德州': 371400,
'聊城': 371500,
'滨州': 371600,
'菏泽': 371700,
},
'河南': {
'河南': 410000,
'郑州': 410100,
'开封': 410200,
'洛阳': 410300,
'平顶山': 410400,
'安阳': 410500,
'鹤壁': 410600,
'新乡': 410700,
'焦作': 410800,
'濮阳': 410900,
'许昌': 411000,
'漯河': 411100,
'三门峡': 411200,
'南阳': 411300,
'商丘': 411400,
'信阳': 411500,
'周口': 411600,
'驻马店': 411700,
'济源': 419001,
},
'湖北': {
'湖北': 420000,
'武汉': 420100,
'黄石': 420200,
'十堰': 420300,
'宜昌': 420500,
'襄阳': 420600,
'鄂州': 420700,
'荆门': 420800,
'孝感': 420900,
'荆州': 421000,
'黄冈': 421100,
'咸宁': 421200,
'随州': 421300,
'恩施土家族苗族自治州': 422800,
'恩施州': 422800,
'仙桃': 429004,
'潜江': 429005,
'天门': 429006,
'神农架林区': 429021,
},
'湖南': {
'湖南': 430000,
'长沙': 430100,
'株洲': 430200,
'湘潭': 430300,
'衡阳': 430400,
'邵阳': 430500,
'岳阳': 430600,
'常德': 430700,
'张家界': 430800,
'益阳': 430900,
'郴州': 431000,
'永州': 431100,
'怀化': 431200,
'娄底': 431300,
'湘西土家族苗族自治州': 433100,
'湘西州': 433100,
},
'广东': {
'广东': 440000,
'广州': 440100,
'韶关': 440200,
'深圳': 440300,
'珠海': 440400,
'汕头': 440500,
'佛山': 440600,
'江门': 440700,
'湛江': 440800,
'茂名': 440900,
'肇庆': 441200,
'惠州': 441300,
'梅州': 441400,
'汕尾': 441500,
'河源': 441600,
'阳江': 441700,
'清远': 441800,
'东莞': 441900,
'中山': 442000,
'潮州': 445100,
'揭阳': 445200,
'云浮': 445300,
},
'广西': {
'广西': 450000,
'南宁': 450100,
'柳州': 450200,
'桂林': 450300,
'梧州': 450400,
'北海': 450500,
'防城港': 450600,
'钦州': 450700,
'贵港': 450800,
'玉林': 450900,
'百色': 451000,
'贺州': 451100,
'河池': 451200,
'来宾': 451300,
'崇左': 451400,
},
'海南': {
'海南': 460000,
'海口': 460100,
'三亚': 460200,
'三沙': 460300,
'儋州': 460400,
'五指山': 469001,
'琼海': 469002,
'文昌': 469005,
'万宁': 469006,
'东方': 469007,
'定安县': 469021,
'屯昌县': 469022,
'澄迈县': 469023,
'临高县': 469024,
'白沙黎族自治县': 469025,
'昌江黎族自治县': 469026,
'乐东黎族自治县': 469027,
'陵水黎族自治县': 469028,
'保亭黎族苗族自治县': 469029,
'琼中黎族苗族自治县': 469030,
},
'重庆': {
'重庆': 500000,
},
'四川': {
'四川': 510000,
'成都': 510100,
'自贡': 510300,
'攀枝花': 510400,
'泸州': 510500,
'德阳': 510600,
'绵阳': 510700,
'广元': 510800,
'遂宁': 510900,
'内江': 511000,
'乐山': 511100,
'南充': 511300,
'眉山': 511400,
'宜宾': 511500,
'广安': 511600,
'达州': 511700,
'雅安': 511800,
'巴中': 511900,
'资阳': 512000,
'阿坝藏族羌族自治州': 513200,
'阿坝州': 513200,
'甘孜藏族自治州': 513300,
'甘孜州': 513300,
'凉山彝族自治州': 513400,
'凉山州': 513400,
},
'贵州': {
'贵州': 520000,
'贵阳': 520100,
'六盘水': 520200,
'遵义': 520300,
'安顺': 520400,
'毕节': 520500,
'铜仁': 520600,
'黔西南布依族苗族自治州': 522300,
'黔西南州': 522300,
'黔东南苗族侗族自治州': 522600,
'黔东南州': 522600,
'黔南布依族苗族自治州': 522700,
'黔南州': 522700,
},
'云南': {
'云南': 530000,
'昆明': 530100,
'曲靖': 530300,
'玉溪': 530400,
'保山': 530500,
'昭通': 530600,
'丽江': 530700,
'普洱': 530800,
'临沧': 530900,
'楚雄彝族自治州': 532300,
'楚雄州': 532300,
'红河哈尼族彝族自治州': 532500,
'红河州': 532500,
'文山壮族苗族自治州': 532600,
'文山州': 532600,
'西双版纳傣族自治州': 532800,
'西双版纳州': 532800,
'大理白族自治州': 532900,
'大理州': 532900,
'德宏傣族景颇族自治州': 533100,
'德宏州': 533100,
'怒江傈僳族自治州': 533300,
'怒江州': 533300,
'迪庆藏族自治州': 533400,
'迪庆州': 533400,
},
'西藏': {
'西藏': 540000,
'拉萨': 540100,
'日喀则': 540200,
'昌都': 540300,
'林芝': 540400,
'山南': 540500,
'那曲': 540600,
'阿里地区': 542500,
},
'陕西': {
'陕西': 610000,
'西安': 610100,
'铜川': 610200,
'宝鸡': 610300,
'咸阳': 610400,
'渭南': 610500,
'延安': 610600,
'汉中': 610700,
'榆林': 610800,
'安康': 610900,
'商洛': 611000,
},
'甘肃': {
'甘肃': 620000,
'兰州': 620100,
'嘉峪关': 620200,
'金昌': 620300,
'白银': 620400,
'天水': 620500,
'武威': 620600,
'张掖': 620700,
'平凉': 620800,
'酒泉': 620900,
'庆阳': 621000,
'定西': 621100,
'陇南': 621200,
'临夏回族自治州': 622900,
'临夏州': 622900,
'甘南藏族自治州': 623000,
'甘南州': 623000,
},
'青海': {
'青海': 630000,
'西宁': 630100,
'海东': 630200,
'海北藏族自治州': 632200,
'海北州': 632200,
'黄南藏族自治州': 632300,
'黄南州': 632300,
'海南藏族自治州': 632500,
'海南州': 632500,
'果洛藏族自治州': 632600,
'果洛州': 632600,
'玉树藏族自治州': 632700,
'玉树州': 632700,
'海西蒙古族藏族自治州': 632800,
'海西州': 632800,
},
'宁夏': {
'宁夏': 640000,
'银川': 640100,
'石嘴山': 640200,
'吴忠': 640300,
'固原': 640400,
'中卫': 640500,
},
'新疆': {
'新疆': 650000,
'乌鲁木齐': 650100,
'克拉玛依': 650200,
'吐鲁番': 650400,
'哈密': 650500,
'昌吉回族自治州': 652300,
'昌吉州': 652300,
'博尔塔拉蒙古自治州': 652700,
'博尔塔拉州': 652700,
'巴音郭楞蒙古自治州': 652800,
'巴音郭楞州': 652800,
'阿克苏地区': 652900,
'克孜勒苏柯尔克孜自治州': 653000,
'克孜勒苏州': 653000,
'和田地区': 653200,
'伊犁哈萨克自治州': 654000,
'伊犁州': 654000,
'塔城地区': 654200,
'阿勒泰地区': 654300,
'石河子': 659001,
'阿拉尔': 659002,
'图木舒克': 659003,
'五家渠': 659004,
'铁门关': 659006,
},
'台湾': {
'台湾': 710000,
},
'香港': {
'香港': 810000,
},
'澳门': {
'澳门': 820000,
},
}

45
src/plugins/cac/index.js Normal file
View File

@@ -0,0 +1,45 @@
const list = require('./data')
const query = (country, region, city) => {
if (country === '中国') {
if (list[region]) {
const result = list[region][city] || list[region][region]
return result.toString()
}
// 尝试前缀匹配
for (const key in list) {
if (region.indexOf(key) === 0) {
if (city) {
const cityList = list[key]
for (const cityKey in cityList) {
if (city.indexOf(cityKey) === 0) {
return cityList[cityKey].toString()
}
}
}
return list[key][key].toString()
}
}
return '100000'
}
// 港澳台地区
if (country === '中国香港') {
return '810000'
}
if (country === '中国澳门') {
return '820000'
}
if (country === '中国台湾') {
return '710000'
}
return ''
}
const patch = data => {
if (!data.china_admin_code) {
data.china_admin_code = query(data.country_name, data.region_name, data.city_name)
}
return data
}
module.exports = patch

38
src/plugins/china.js Normal file
View File

@@ -0,0 +1,38 @@
const cidrTools = require('cidr-tools')
const plugin = (through2, file, cb) => {
let data = file.contents.toString().trim().split('\n')
let white = `
1.8.0.0/16
1.24.0.0/13
52.80.0.0/14
61.128.0.0/10
112.0.0.0/10
119.90.0.0/15
121.59.0.0/16
124.250.192.0/18
128.108.0.0/16
129.28.0.0/16
159.226.0.0/16
182.254.0.0/17
202.96.0.0/12
210.52.0.0/15
211.64.0.0/13
211.136.0.0/13
218.104.0.0/14
219.90.68.0/22
219.90.72.0/21
219.128.0.0/11
223.120.0.0/13
`
white = white.trim().split('\n')
data = cidrTools.merge([...data, ...white])
file.contents = Buffer.from(data.join('\n'))
file.path = file.path.replace('CN.txt', 'china.txt')
cb(null, file)
}
module.exports = plugin

29
src/plugins/cidrmerge.js Normal file
View File

@@ -0,0 +1,29 @@
const cidrTools = require('cidr-tools')
const ProgressBar = require('progress')
const plugin = (file, _, cb) => {
let cidrs = file.contents.toString().split('\n')
console.log(file.path)
if (cidrs.length) {
let result = [cidrs[0]]
let bar = new ProgressBar(':bar :current/:total', { total: cidrs.length })
for (let i = 1; i < cidrs.length; i += 1) {
let t = cidrTools.merge([result[result.length - 1], cidrs[i]])
if (t.length === 1) {
result[result.length - 1] = t[0]
} else {
result.push(cidrs[i])
}
bar.tick()
}
cidrs = result
console.log()
}
file.contents = Buffer.from(cidrs.join('\n'))
cb(null, file)
}
module.exports = plugin

47
src/plugins/city.js Normal file
View File

@@ -0,0 +1,47 @@
const vinyl = require('vinyl')
const IPDB = require('ipdb')
const ipdb_range = require('@ipdb/range')
const ProgressBar = require('progress')
const plugin = (through2, file, cb) => {
console.log('Parse ipdb')
const ipdb = new IPDB(file.contents, {
patches: [ipdb_range]
})
let bar = new ProgressBar(':bar :current/:total', { total: ipdb.meta.node_count })
let result = []
let ip = '0.0.0.0'
while (true) {
const info = ipdb.find(ip).data
const iso3166_2 = info.region_code
if (iso3166_2.length > 3) {
if (!result[iso3166_2]) {
result[iso3166_2] = []
}
result[iso3166_2].push(`${info.range.from}/${info.bitmask}`)
}
bar.tick()
ip = info.range.next
if (ip === '0.0.0.0') break
}
console.log()
for (let [country_code, cidrs] of Object.entries(result)) {
let temp = new vinyl({
cwd: '/',
base: '/',
path: `/${country_code.split('-')[0]}/${country_code}.txt`,
contents: new Buffer.from(cidrs.join('\n'))
})
through2.push(temp)
}
cb()
}
module.exports = plugin

59
src/plugins/cncity.js Normal file
View File

@@ -0,0 +1,59 @@
const vinyl = require('vinyl')
const IPDB = require('ipdb')
const ipdb_range = require('@ipdb/range')
const ipdb_cac = require('./cac')
const ProgressBar = require('progress')
const plugin = (through2, file, cb) => {
console.log('Parse ipdb')
const ipdb = new IPDB(file.contents, {
patches: [ipdb_range, ipdb_cac]
})
let bar = new ProgressBar(':bar :current/:total', { total: ipdb.meta.node_count })
let result = []
let ip = '0.0.0.0'
while (true) {
const info = ipdb.find(ip).data
const china_admin_code = info.china_admin_code
if (china_admin_code?.length === 6) {
let cac = china_admin_code
{
cac = `${cac.substr(0, 4)}00`
if (!result[cac]) {
result[cac] = []
}
result[cac].push(`${info.range.from}/${info.bitmask}`)
}
{
cac = `${cac.substr(0, 2)}0000`
if (!result[cac]) {
result[cac] = []
}
result[cac].push(`${info.range.from}/${info.bitmask}`)
}
}
bar.tick()
ip = info.range.next
if (ip === '0.0.0.0') break
}
console.log()
for (let [china_admin_code, cidrs] of Object.entries(result)) {
let temp = new vinyl({
cwd: '/',
base: '/',
path: `/${china_admin_code}.txt`,
contents: new Buffer.from(cidrs.join('\n'))
})
through2.push(temp)
}
cb()
}
module.exports = plugin

47
src/plugins/country.js Normal file
View File

@@ -0,0 +1,47 @@
const vinyl = require('vinyl')
const IPDB = require('ipdb')
const ipdb_range = require('@ipdb/range')
const ProgressBar = require('progress')
const plugin = (through2, file, cb) => {
console.log('Parse ipdb')
const ipdb = new IPDB(file.contents, {
patches: [ipdb_range]
})
let bar = new ProgressBar(':bar :current/:total', { total: ipdb.meta.node_count })
let result = []
let ip = '0.0.0.0'
while (true) {
const info = ipdb.find(ip).data
const iso3166_1 = info.country_code
if (iso3166_1.length === 2) {
if (!result[iso3166_1]) {
result[iso3166_1] = []
}
result[iso3166_1].push(`${info.range.from}/${info.bitmask}`)
}
bar.tick()
ip = info.range.next
if (ip === '0.0.0.0') break
}
console.log()
for (let [country_code, cidrs] of Object.entries(result)) {
let temp = new vinyl({
cwd: '/',
base: '/',
path: `/${country_code}.txt`,
contents: new Buffer.from(cidrs.join('\n'))
})
through2.push(temp)
}
cb()
}
module.exports = plugin