DEV

-continue implementation

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ChaChaIPToCountryDaemon</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

ChaChaIPToCountryDaemon/__init__.py

@@ -0,0 +1,13 @@
+"""
+ChaChaIPToCountryDaemon (c) by clement chastanier
+
+ChaChaIPToCountryDaemon  is licensed under a
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
+
+You should have received a copy of the license along with this
+work.  If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
+"""
+
+from ._version import __version__
+
+from .core import   ChaChaIPToCountryDaemon

ChaChaIPToCountryDaemon/_version.py

@@ -0,0 +1,11 @@
+"""
+ChaChaIPToCountryDaemon (c) by clement chastanier
+
+ChaChaIPToCountryDaemon  is licensed under a
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
+
+You should have received a copy of the license along with this
+work.  If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
+"""
+
+__version__ = "0.1"

ChaChaIPToCountryDaemon/core.py

@@ -0,0 +1,215 @@
+"""
+ChaChaIPToCountryDaemon (c) by clement chastanier
+
+ChaChaIPToCountryDaemon  is licensed under a
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
+
+You should have received a copy of the license along with this
+work.  If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
+"""
+
+
+from typing import Union
+import tornado.ioloop
+import tornado.web
+from tornado.httpclient import AsyncHTTPClient
+import json
+import tempfile
+from git import Repo
+import os
+from pathlib import Path
+from netaddr import IPSet,IPAddress
+import asyncio
+from concurrent.futures import ProcessPoolExecutor
+from ipaddress import ip_address, IPv4Address
+import pycountry
+from datetime import datetime, timedelta
+from pytz import timezone 
+from functools import wraps
+from asyncstdlib import lru_cache 
+from dns import resolver,reversename
+
+DEFAULT__Cache_Expiration_Second            = 600
+DEFAULT__Cache_Size_Entries                 = 2048
+DEFAULT__IpDataSet_UpdatePeriod_Second      = 120
+DEFAULT__Num_IpDataSet_Workers              = 10
+DEFAULT__IpDataSet_GitRepo_Address          = "https://chacha.ddns.net/gitea/chacha/country-ip-blocks"
+DEFAULT__IpDataSet_ipv4_subdir              = "ipv4"
+DEFAULT__IpDataSet_ipv6_subdir              = "ipv6"
+
+def processfile(_entry_path,_entry_code) :
+    with open(_entry_path) as fp:
+        _set = IPSet()
+        for line in  fp:
+            _set.add(line)   
+    print("PARSE DONE({0})".format(_entry_path))
+    return _entry_code,_set
+
+def validIPAddress(IP: str) -> str:
+    try:
+        return "IPv4" if type(ip_address(IP)) is IPv4Address else "IPv6"
+    except ValueError:
+            return "Invalid"
+
+def timed_lru_cache(seconds: int, maxsize: int = None):
+    def wrapper_cache(func):
+        func = lru_cache(maxsize=maxsize)(func)
+        func.lifetime = timedelta(seconds=seconds)
+        func.expiration = datetime.utcnow() + func.lifetime
+        @wraps(func)
+        def wrapped_func(*args, **kwargs):
+            if datetime.utcnow() >= func.expiration:
+                func.cache_clear()
+                func.expiration = datetime.utcnow() + func.lifetime
+            return func(*args, **kwargs)
+        return wrapped_func
+    return wrapper_cache
+        
+class ChaChaIPToCountryStorage:
+    def __init__(self):
+        self.ipv4set        = dict()
+        self.ipv6set        = dict()
+        self.tempdir        = tempfile.TemporaryDirectory()
+        self.tempdir_url    = self.tempdir.name  + os.sep
+        self.ipv4_dir       = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv4_subdir)
+        self.ipv6_dir       = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv6_subdir) 
+        self.DataSetlock    = asyncio.Lock()
+        print("New dataset dir = " + self.tempdir_url)
+        print("Cloning GIT repos: "+ DEFAULT__IpDataSet_GitRepo_Address)
+        self.gitrepo = Repo.clone_from(DEFAULT__IpDataSet_GitRepo_Address,self.tempdir_url )
+        print("Done")
+        asyncio.get_event_loop().run_until_complete(self.update_repo())        
+        print("Finished")
+        
+    @timed_lru_cache(seconds=DEFAULT__Cache_Expiration_Second,maxsize=DEFAULT__Cache_Size_Entries)
+    async def test_ip(self,ip):
+        validity = validIPAddress(ip)
+        
+        if validity == "IPv4":
+            set =self.ipv4set
+        elif validity == "IPv6":
+            set =self.ipv6set
+        else:  
+            RuntimeError("Invalid ip address received")
+            
+        await self.DataSetlock.acquire()
+        try:
+            for key,set in set.items():            
+                if IPAddress(ip) in set:
+                    return key
+        finally:
+            self.DataSetlock.release()
+            
+        return "ZZ"
+    
+    async def update_repo(self):
+        print("== DATASET UPDATE REQUESTED ==")
+        print("Pulling GIT dataset repo")
+        o = self.gitrepo.remotes.origin
+        o.pull()
+        print("Done")
+        
+        tasksIPV4 = []
+        tasksIPV6 = []
+        loop = asyncio.get_event_loop()
+        _executor = ProcessPoolExecutor(DEFAULT__Num_IpDataSet_Workers)  
+              
+        print("Parsing IPV4 dataset")
+        with os.scandir(self.ipv4_dir) as entries:
+            for entry in entries:
+                entry_path = os.path.join(self.ipv4_dir,entry.name)
+                entry_code = Path(entry_path).stem  
+                if entry_code not in ["fr"]: continue          
+                task = loop.run_in_executor(_executor,processfile,entry_path,entry_code)
+                tasksIPV4.append(task)
+        print("Done")
+        
+        #print("Parsing IPV6 dataset")
+        #with os.scandir(self.ipv6_dir) as entries:
+        #    for entry in entries:
+        #        entry_path = os.path.join(self.ipv6_dir,entry.name)
+        #        entry_code = Path(entry_path).stem 
+        #        if entry_code not in ["fr"]: continue            
+        #        task = loop.run_in_executor(_executor,processfile,entry_path,entry_code)
+        #        tasksIPV6.append(task) 
+        #print("Done")
+                  
+        print("Wait IPV4 parsing to complete...")
+        results = await asyncio.gather(*tasksIPV4)
+        _ipv4set=dict()
+        for _entry_code,_set in results:
+            _ipv4set[_entry_code] = _set            
+        print("Done")
+        
+        print("Updating live IPV4 storage")
+        await self.DataSetlock.acquire()
+        try:
+            self.ipv4set = _ipv4set
+        finally:
+            self.DataSetlock.release()
+        print("Done")
+        
+        print("Wait IPV6 parsing to complete...")
+        results = await asyncio.gather(*tasksIPV6)
+        _ipv6set=dict()
+        for _entry_code,_set in results:
+            _ipv6set[_entry_code] = _set            
+        print("Done")        
+        
+        print("Updating live IPV6 storage")  
+        await self.DataSetlock.acquire()
+        try:   
+            self.ipv6set = _ipv6set
+        finally:
+            self.DataSetlock.release()
+        print("Done")
+        print("== DATASET UPDATE FINISHED ==")
+        
+class ChaChaIPToCountryDaemon:
+    def __init__(self,serverport:int=80,baseurl:str=""):
+        self.storage = ChaChaIPToCountryStorage()
+        self.handler = tornado.web.Application([
+            (baseurl + r"/api/ip2country/", ChaChaIPToCountryDaemon_Handler_REST,   {'storage':self.storage}),    # new RESTfull API handler (POST)
+            (baseurl + r"/ip2country.php",  ChaChaIPToCountryDaemon_Handler_Legacy, {'storage':self.storage}),  # legacy php-like handler (GET)
+        ])
+        self.handler.listen(serverport)
+        ioloop = tornado.ioloop.IOLoop.current()
+        update_cb = tornado.ioloop.PeriodicCallback(self.storage.update_repo, 1000*DEFAULT__IpDataSet_UpdatePeriod_Second)
+        update_cb.start()
+        ioloop.start()    
+
+class ChaChaIPToCountryDaemon_Handler(tornado.web.RequestHandler):      
+    def initialize(self, storage): 
+        self.storage = storage 
+        
+    async def getIP2Country_Full(self,ip):
+        result = dict()
+        result["alpha_2"] = await self.getIP2Country(ip)        
+        country = pycountry.countries.get(alpha_2=result["alpha_2"])
+        result["coutry_name"]=country.name.upper()        
+        result["alpha_3"]=country.alpha_3        
+        result["coutry_official_name"]=country.official_name        
+        result["utc_time"]= datetime.now(timezone("UTC")).strftime('%H:%M')
+        result["ip"]=ip
+        addr=reversename.from_address(ip)
+        result["dns"]=str(resolver.query(addr,"PTR")[0])[:-1]
+        return result
+        
+    async def getIP2Country(self,ip):
+        print("Requested ip address: {0}".format(ip))
+        return await self.storage.test_ip(ip)
+    
+class ChaChaIPToCountryDaemon_Handler_REST(ChaChaIPToCountryDaemon_Handler):
+    async def post(self):
+        ipToCompute = json.loads(self.request.body)   
+        res = await self.getIP2Country_Full(ipToCompute["ip"])
+        self.write(json.dumps(res).encode("utf-8"))
+        self.finish()
+            
+class ChaChaIPToCountryDaemon_Handler_Legacy(ChaChaIPToCountryDaemon_Handler):
+    async def get(self):
+        ipToCompute = self.get_argument('ip')      
+        res = await self.getIP2Country_Full(ipToCompute)
+        resline= "{0} {1}:{2}:{3}:{4}:{5}".format(res["utc_time"],res["ip"],res["dns"],res["coutry_name"],res["alpha_3"],res["alpha_2"])
+        self.write(resline.encode("utf-8"))
+        self.finish()

Tests/TEST_base.py

@@ -0,0 +1,20 @@
+from pprint import pprint
+from pathlib import Path
+
+import unittest
+
+import sys
+import io
+sys.path.append('../')
+
+from ChaChaIPToCountryDaemon import *
+
+
+class Test_ChaChaIPToCountryDaemon_base(unittest.TestCase):
+
+    def setUp(self):        
+        [f.unlink() for f in Path("tmp").glob("*")] 
+        print("======================")
+  
+    def test_simplerun(self):
+        tmp = ChaChaIPToCountryDaemon()

setup.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from setuptools import setup,find_packages
+from ChaChaIPToCountryDaemon import __version__ as ChaChaIPToCountryDaemon_Version
+
+with open("./README.md", "r") as fh:
+    long_description = fh.read()
+    
+setup(
+    name='ChaChaIPToCountryDaemon',
+    version=ChaChaIPToCountryDaemon_Version,
+    description='An HTTP/REST IP2Country Daemon using country-ip-blocks database.',
+    author='Clement CHASTANIER',
+    author_email='clement.chastanier@gmail.com',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url='https://chacha.ddns.net/gitea/chacha/ChaChaIPToCountryDaemon',
+    packages=find_packages(),
+    install_requires=["tornado","gitpython","netaddr","aiofiles","pycountry","dnspython","asyncstdlib"],
+    license="CC BY-NC-SA 4.0",
+    python_requires='>=3.8',
+     ),
+