初始化运行代码

.gitignore

@@ -0,0 +1 @@
+*.awdb

analyze-v6.py

@@ -0,0 +1,72 @@
+import pandas as pd
+from rich.progress import Progress
+
+from analyzer import node
+
+data = []
+with Progress() as p:
+    # inputdata = "./openIPv6dns.txt"
+    inputdata="./recurisivev6dns.txt"
+    print(len(open(inputdata).readlines()))
+    task = p.add_task("[blue]Working...", total=sum(1 for _ in open(inputdata)))
+    # exclist=["中国台湾","中国香港"]
+    with open(inputdata, "r") as f:
+        while True:
+            l = f.readline()
+            if l == "":
+                break
+            ip6 = l.splitlines()[0]
+            n = node(ip6)
+            p.update(task, advance=1, refresh=True)
+            # if n.prov=="UnKnown":
+            if n.isp=="UnKnown":
+                continue
+            # if n.cou=="中国" and n.prov not in exclist and "澳" not in n.isp:
+            data.append({"ip": n.ip,
+                        "owner": n.owner,
+                        "isp": n.isp,
+                        "area": n.prov,
+                        "asn": n.AS,
+                        "lat": n.lat,
+                        "lng": n.lng,
+                        "coucode":n.couCode,
+                        "cou":n.cou})
+d = pd.DataFrame(data)
+import pygal 
+# 地图可视化
+#创建一个字典
+print(d.shape[0])
+cc = d['isp'].value_counts().to_dict()
+# cc = d['area'].value_counts().to_dict()
+# from pygal_maps_world.i18n import COUNTRIES
+# def get_cc(ccname):
+#     #根据指定的国家，返回Pygal使用的两个字母的国别码
+#     for code,name in COUNTRIES.items():
+#         if ccname in name :
+#             return code
+#     # 如果没有找到指定的国家，就返回None
+#     return None
+
+# cc_stats={}
+# for pop_dic,pop_v in cc.items() :
+#     country_name= pop_dic
+#     code =  get_cc(country_name)
+#     if code :
+#         cc_stats[code] =  pop_v
+# import pygal.maps.world#创建了一个Worldmap实例，并设置了该地图的的title属性
+# wm = pygal.maps.world.World(print_values=True)
+# wm.title = '全球递归IPv6 DNS解析器分布'
+# wm.add('IPv6 DNS数量', cc_stats)
+# wm.render_to_file('world_v6Rdns.svg')
+
+# 柱状图
+bar_chart = pygal.Bar(print_values=True, print_values_position='top')
+# bar_chart.title = '国内各服务商开放IPv6 DNS解析器数量'
+i=0
+x=[]
+for k,v in cc.items():
+    bar_chart.add(k,v)
+    i+=1
+    if i==20:
+        break
+bar_chart.render_to_file("isp_v6dns.svg")

analyzedutil.py

@@ -0,0 +1,69 @@
+# analyzeutl 是分析器工具集合
+import awdb
+
+
+# 过滤所有空值
+def filterNull(s):
+    if s == "":
+        return "UnKnown"
+    return s
+
+
+def IP46(IP: str):
+    if ':' in IP:
+        return "v6"
+    if '.' in IP:
+        return "v4"
+    return "Unknown"
+
+
+path_ip4app = "./data/IP_scene_all_cn.awdb"
+path_ip6 = "./data/IP_city_single_BD09_WGS84_ipv6.awdb"
+path_ip4qvxian = "./data/IP_basic_single_WGS84.awdb"
+
+
+# 实例化数据读取器
+def makereader(arg=0):
+    # 默认加载所有离线数据
+    dloader_ip4app = awdb.open_database(path_ip4app)
+    dloader_ip6 = awdb.open_database(path_ip6)
+    dloader_ip4qx = awdb.open_database(path_ip4qvxian)
+    return dloader_ip4app, dloader_ip4qx, dloader_ip6
+
+
+reader_ip4app, reader_ip4qx, reader_ip6 = makereader()
+
+
+# # 传入一个ip地址，根据对象中的数据进行IP关联分析
+# def make_IPinfo(ip):
+#     record=getrecord(ip)
+#     cou = record.get('areacode', b'').decode("utf-8")
+#     multiarea = record.get('multiAreas', {})
+#     if multiarea:
+#         print("有多组记录")
+#     else:
+#         pass
+#     return record
+
+# 返回IP离线库中与ip相关的记录，记录中不包含应用场景
+def getrecord(ip):
+    if (IP46(ip) == "v4"):
+        return IP4_info(ip)
+    elif (IP46(ip) == "v6"):
+        return IP6_info(ip)
+    else:
+        print("地址存在问题")
+        print(ip)
+        return 1
+
+
+# 返回IPv4记录
+def IP4_info(ip):
+    (record, prefix_len) = reader_ip4qx.get_with_prefix_len(ip)
+    return record
+
+
+# 返回IPv6记录
+def IP6_info(ip):
+    (record, prefix_len) = reader_ip6.get_with_prefix_len(ip)
+    return record

analyzer.py

@@ -0,0 +1,245 @@
+# !coding=utf-8
+import datetime
+import logging
+import time
+from concurrent import futures
+import grpc
+import pytz
+from neomodel import db, StringProperty, DateTimeFormatProperty, RelationshipTo, StructuredRel, IntegerProperty, \
+    StructuredNode, config, BooleanProperty
+
+import analyze_pb2
+import analyze_pb2_grpc
+import analyzedutil as aul
+
+logging.basicConfig(level=logging.DEBUG,filename="./log/pythonrun.log",filemode="w",
+                    format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')
+
+
+class node:
+    ip = ""
+    AS = ""
+    next = ""
+    isp = ""
+    cou = ""
+    couCode = ""
+    prov = ""
+    lat = ""
+    lng = ""
+    FindTime = ""
+    dataOK = ""
+    owner = ""
+
+    def __init__(self, ip):
+        self.ip = ip
+        record = aul.getrecord(ip)
+        if record == 1:
+            self.dataOK = False
+            return
+        self.dataOK = True
+        self.isp = aul.filterNull(record.get('isp', b'').decode("utf-8"))
+        self.lat = aul.filterNull(record.get('latwgs', b'').decode("utf-8"))
+        self.lng = aul.filterNull(record.get('lngwgs', b'').decode("utf-8"))
+        self.prov = aul.filterNull(record.get('province', b'').decode("utf-8"))
+        self.AS = aul.filterNull(record.get('asnumber', b'').decode("utf-8"))
+        self.couCode = aul.filterNull(record.get('areacode', b'').decode("utf-8"))
+        self.cou = aul.filterNull(record.get('country', b'').decode("utf-8"))
+        self.FindTime = datetime.datetime.now(pytz.UTC)
+        self.owner = aul.filterNull(record.get('owner', b'').decode("utf-8"))
+
+
+# 与go之间的通信
+
+class RequestServe(analyze_pb2_grpc.GrpcServiceServicer):
+    graph_conn = ""
+
+    def AnalyzeService(self, request, context):
+        '''
+        具体实现AnalyzeService服务方法
+        :param request:
+        :param context:
+        :return:
+        '''
+        r = request
+        if r.gtype == "neo4j":
+            if self.graph_conn == "":
+                url = str(r.guri).split("//")[0] + "//" + r.guser + ":" + r.gpass + "@" + str(r.guri).split("//")[1]
+                self.graph_conn = neo4j_connector(url)
+                logging.info("已连接到图数据库Neo4j:" + r.guri)
+            result = self.graph_conn.work_with_neoj_53(r.data)
+            return analyze_pb2.result(res=result)
+        return analyze_pb2.result(res="not support")
+
+
+working_addr = "127.0.0.1"
+working_port = "56789"
+
+
+def serve():
+    # 启动 rpc 服务，这里可定义最大接收和发送大小(单位M)，默认只有4M
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=[
+        ('grpc.max_send_message_length', 100 * 1024 * 1024),
+        ('grpc.max_receive_message_length', 100 * 1024 * 1024)])
+
+    analyze_pb2_grpc.add_GrpcServiceServicer_to_server(RequestServe(), server)
+    server.add_insecure_port(working_addr + ":" + working_port)
+    server.start()
+    logging.info("Python分析模块启动，工作在 " + working_addr + ":" + working_port)
+    try:
+        while True:
+            time.sleep(60 * 60 * 24)  # one day in seconds
+    except KeyboardInterrupt:
+        server.stop(0)
+
+
+class RelResolver53(StructuredRel):
+    W = IntegerProperty()
+    LTIME = DateTimeFormatProperty(default_now=True, format="%Y-%m-%d %H:%M:%S")
+
+# 查询记录定义
+class NodeResolverQuery(StructuredNode):
+    QNAME=StringProperty(required=True)
+    QTYPE=StringProperty()
+# 解析器和查询记录的关系
+class RelResolverQuery(StructuredRel):
+    W = IntegerProperty()
+
+class NodeResolver53(StructuredNode):
+    IP = StringProperty(required=True, unique_index=True)
+    AS = StringProperty()
+    ISP = StringProperty()
+    COU = StringProperty()
+    CCODE = StringProperty()
+    PROV = StringProperty()
+    LAT = StringProperty()
+    LNG = StringProperty()
+    IPType = StringProperty()
+    FTIME = DateTimeFormatProperty(format="%Y-%m-%d %H:%M:%S")
+    LTIME = DateTimeFormatProperty(default_now=True, format="%Y-%m-%d %H:%M:%S")
+    W = IntegerProperty()
+    ISPUBLIC = BooleanProperty(default=False)
+    LINK = RelationshipTo("NodeResolver53", "IP_LINK", model=RelResolver53)
+    QLINK=RelationshipTo("NodeResolverQuery","Q_LINK",model=RelResolverQuery)
+    
+
+class neo4j_connector:
+    graph = ""
+
+    # nodematcher = ""
+    # relatmatcher = ""
+
+    def __init__(self, url):
+        # 连接neo4j
+        #config.ENCRYPTED = True
+        config.DATABASE_URL =url
+        db.set_connection(url)
+#  data=[ip1,ip2,ispublic,qname,qtype]
+    def work_with_neoj_53(self, data):
+        datastr=""
+        for d in data:
+            datastr=datastr+str(d)+" , "
+        logging.debug("处理数据："+datastr+"")
+        ############################################### 对解析器节点进行处理#####################################################
+        for d in range(len(data) - 3):
+            n = node(data[d])
+            if not n.dataOK:
+                return "node err because ip"
+            # 查询是否存在节点
+            nd, exist = self.checknode_neo4j(ip=n.ip)
+            # 不存在则新建
+            if not exist:
+                nd = NodeResolver53(AS=n.AS, COU=n.cou,
+                                    CCODE=n.couCode, LAT=n.lat, LNG=n.lng,
+                                    ISP=n.isp, IPType=aul.IP46(n.ip), PROV=n.prov, FTIME=n.FindTime,
+                                    LTIME=n.FindTime, IP=n.ip, W=1)
+                # IP1是开放解析器
+                if data[2] == "0" and d == 0:
+                    nd.ISPUBLIC = True
+                nd.save()
+            # 存在则只修改时间
+            else:
+                # nd.LTIME = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+                nd.LTIME = datetime.datetime.now(pytz.UTC)
+                if nd.W is not None:
+                    nd.W += 1
+                else:
+                    nd.W = 1
+                nd.save()
+                
+                    
+                    
+        ################################################ 对查询记录节点进行处理################################################
+        # 查询是否存在节点
+        # q,exist=self.checknode_neo4j(q=data[3],qtype=data[4])
+        # # 不存在则新建
+        # if not exist:
+        #     q = NodeResolverQuery(QNAME=data[3],QTYPE=data[4])
+        #     q.save()
+        # 存在则不做处理
+        
+
+        ############################################ 查询解析器是否存在关系#############################################
+        L, lexist = self.checklink_neo4j(data[0], data[1])
+        # 数据存在问题则退出
+        if L == "Err":
+            return "node err when link"
+        # 不存在则建立关联
+        if not lexist:
+            L[0].LINK.connect(L[1], {'W': 1, 'LTIME': datetime.datetime.now(pytz.UTC)}).save()
+        # 存在则修改权重
+        else:
+            L.W += 1
+            L.LTIME = datetime.datetime.now(pytz.UTC)
+            L.save()
+        # 提交链接
+        
+        ############################################查询解析器和记录间的关系#########################################
+        # QL, lexist = self.checkquerylink(data[1], data[3],data[4])
+        # # 数据存在问题则退出
+        # if QL == "Err":
+        #     return "node err when link"
+        # # 不存在则建立关联
+        # if not lexist:
+        #     QL[0].QLINK.connect(QL[1], {'W': 1}).save()
+        # # 存在则修改权重
+        # else:
+        #     QL.W += 1
+        #     QL.save()
+            
+        # 完成处理，返回
+        logging.debug("完成处理数据：{"+datastr+"}")
+        return "success"
+
+    def checknode_neo4j(self, ip=None,q=None,qtype=None):
+        # 查询IP
+        if ip!=None:
+            a = NodeResolver53.nodes.get_or_none(IP=ip)
+        # 查询记录
+        else:
+            a=NodeResolverQuery.nodes.get_or_none(QNAME=q,QTYPE=qtype)
+        if a is not None:
+            return a, True
+        return None, False
+
+    def checklink_neo4j(self, ip_from, ip_to):
+        f = NodeResolver53.nodes.get_or_none(IP=ip_from)
+        t = NodeResolver53.nodes.get_or_none(IP=ip_to)
+        if f is None or t is None:
+            return "Err", False
+        rel = f.LINK.relationship(t)
+        if rel is not None:
+            return rel, True
+        return [f, t], False
+    
+    def checkquerylink(self,ip,qname,qtype):
+        r=NodeResolver53.nodes.get_or_none(IP=ip)
+        q=NodeResolverQuery.nodes.get_or_none(QNAME=qname,QTYPE=qtype)
+        if r is None or q is None:
+            return "Err", False
+        rel=r.QLINK.relationship(q)
+        if rel is not None:
+            return rel, True
+        return [r, q], False
+
+if __name__ == '__main__':
+    serve()