diff --git a/.project b/.project new file mode 100644 index 0000000..3d3fb32 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + ChaChaIPToCountryDaemon + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/ChaChaIPToCountryDaemon/__init__.py b/ChaChaIPToCountryDaemon/__init__.py new file mode 100644 index 0000000..b8a8c54 --- /dev/null +++ b/ChaChaIPToCountryDaemon/__init__.py @@ -0,0 +1,13 @@ +""" +ChaChaIPToCountryDaemon (c) by clement chastanier + +ChaChaIPToCountryDaemon is licensed under a +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License. + +You should have received a copy of the license along with this +work. If not, see . +""" + +from ._version import __version__ + +from .core import ChaChaIPToCountryDaemon \ No newline at end of file diff --git a/ChaChaIPToCountryDaemon/_version.py b/ChaChaIPToCountryDaemon/_version.py new file mode 100644 index 0000000..a38b449 --- /dev/null +++ b/ChaChaIPToCountryDaemon/_version.py @@ -0,0 +1,11 @@ +""" +ChaChaIPToCountryDaemon (c) by clement chastanier + +ChaChaIPToCountryDaemon is licensed under a +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License. + +You should have received a copy of the license along with this +work. If not, see . +""" + +__version__ = "0.1" \ No newline at end of file diff --git a/ChaChaIPToCountryDaemon/core.py b/ChaChaIPToCountryDaemon/core.py new file mode 100644 index 0000000..3880cd2 --- /dev/null +++ b/ChaChaIPToCountryDaemon/core.py @@ -0,0 +1,215 @@ +""" +ChaChaIPToCountryDaemon (c) by clement chastanier + +ChaChaIPToCountryDaemon is licensed under a +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License. + +You should have received a copy of the license along with this +work. If not, see . +""" + + +from typing import Union +import tornado.ioloop +import tornado.web +from tornado.httpclient import AsyncHTTPClient +import json +import tempfile +from git import Repo +import os +from pathlib import Path +from netaddr import IPSet,IPAddress +import asyncio +from concurrent.futures import ProcessPoolExecutor +from ipaddress import ip_address, IPv4Address +import pycountry +from datetime import datetime, timedelta +from pytz import timezone +from functools import wraps +from asyncstdlib import lru_cache +from dns import resolver,reversename + +DEFAULT__Cache_Expiration_Second = 600 +DEFAULT__Cache_Size_Entries = 2048 +DEFAULT__IpDataSet_UpdatePeriod_Second = 120 +DEFAULT__Num_IpDataSet_Workers = 10 +DEFAULT__IpDataSet_GitRepo_Address = "https://chacha.ddns.net/gitea/chacha/country-ip-blocks" +DEFAULT__IpDataSet_ipv4_subdir = "ipv4" +DEFAULT__IpDataSet_ipv6_subdir = "ipv6" + +def processfile(_entry_path,_entry_code) : + with open(_entry_path) as fp: + _set = IPSet() + for line in fp: + _set.add(line) + print("PARSE DONE({0})".format(_entry_path)) + return _entry_code,_set + +def validIPAddress(IP: str) -> str: + try: + return "IPv4" if type(ip_address(IP)) is IPv4Address else "IPv6" + except ValueError: + return "Invalid" + +def timed_lru_cache(seconds: int, maxsize: int = None): + def wrapper_cache(func): + func = lru_cache(maxsize=maxsize)(func) + func.lifetime = timedelta(seconds=seconds) + func.expiration = datetime.utcnow() + func.lifetime + @wraps(func) + def wrapped_func(*args, **kwargs): + if datetime.utcnow() >= func.expiration: + func.cache_clear() + func.expiration = datetime.utcnow() + func.lifetime + return func(*args, **kwargs) + return wrapped_func + return wrapper_cache + +class ChaChaIPToCountryStorage: + def __init__(self): + self.ipv4set = dict() + self.ipv6set = dict() + self.tempdir = tempfile.TemporaryDirectory() + self.tempdir_url = self.tempdir.name + os.sep + self.ipv4_dir = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv4_subdir) + self.ipv6_dir = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv6_subdir) + self.DataSetlock = asyncio.Lock() + print("New dataset dir = " + self.tempdir_url) + print("Cloning GIT repos: "+ DEFAULT__IpDataSet_GitRepo_Address) + self.gitrepo = Repo.clone_from(DEFAULT__IpDataSet_GitRepo_Address,self.tempdir_url ) + print("Done") + asyncio.get_event_loop().run_until_complete(self.update_repo()) + print("Finished") + + @timed_lru_cache(seconds=DEFAULT__Cache_Expiration_Second,maxsize=DEFAULT__Cache_Size_Entries) + async def test_ip(self,ip): + validity = validIPAddress(ip) + + if validity == "IPv4": + set =self.ipv4set + elif validity == "IPv6": + set =self.ipv6set + else: + RuntimeError("Invalid ip address received") + + await self.DataSetlock.acquire() + try: + for key,set in set.items(): + if IPAddress(ip) in set: + return key + finally: + self.DataSetlock.release() + + return "ZZ" + + async def update_repo(self): + print("== DATASET UPDATE REQUESTED ==") + print("Pulling GIT dataset repo") + o = self.gitrepo.remotes.origin + o.pull() + print("Done") + + tasksIPV4 = [] + tasksIPV6 = [] + loop = asyncio.get_event_loop() + _executor = ProcessPoolExecutor(DEFAULT__Num_IpDataSet_Workers) + + print("Parsing IPV4 dataset") + with os.scandir(self.ipv4_dir) as entries: + for entry in entries: + entry_path = os.path.join(self.ipv4_dir,entry.name) + entry_code = Path(entry_path).stem + if entry_code not in ["fr"]: continue + task = loop.run_in_executor(_executor,processfile,entry_path,entry_code) + tasksIPV4.append(task) + print("Done") + + #print("Parsing IPV6 dataset") + #with os.scandir(self.ipv6_dir) as entries: + # for entry in entries: + # entry_path = os.path.join(self.ipv6_dir,entry.name) + # entry_code = Path(entry_path).stem + # if entry_code not in ["fr"]: continue + # task = loop.run_in_executor(_executor,processfile,entry_path,entry_code) + # tasksIPV6.append(task) + #print("Done") + + print("Wait IPV4 parsing to complete...") + results = await asyncio.gather(*tasksIPV4) + _ipv4set=dict() + for _entry_code,_set in results: + _ipv4set[_entry_code] = _set + print("Done") + + print("Updating live IPV4 storage") + await self.DataSetlock.acquire() + try: + self.ipv4set = _ipv4set + finally: + self.DataSetlock.release() + print("Done") + + print("Wait IPV6 parsing to complete...") + results = await asyncio.gather(*tasksIPV6) + _ipv6set=dict() + for _entry_code,_set in results: + _ipv6set[_entry_code] = _set + print("Done") + + print("Updating live IPV6 storage") + await self.DataSetlock.acquire() + try: + self.ipv6set = _ipv6set + finally: + self.DataSetlock.release() + print("Done") + print("== DATASET UPDATE FINISHED ==") + +class ChaChaIPToCountryDaemon: + def __init__(self,serverport:int=80,baseurl:str=""): + self.storage = ChaChaIPToCountryStorage() + self.handler = tornado.web.Application([ + (baseurl + r"/api/ip2country/", ChaChaIPToCountryDaemon_Handler_REST, {'storage':self.storage}), # new RESTfull API handler (POST) + (baseurl + r"/ip2country.php", ChaChaIPToCountryDaemon_Handler_Legacy, {'storage':self.storage}), # legacy php-like handler (GET) + ]) + self.handler.listen(serverport) + ioloop = tornado.ioloop.IOLoop.current() + update_cb = tornado.ioloop.PeriodicCallback(self.storage.update_repo, 1000*DEFAULT__IpDataSet_UpdatePeriod_Second) + update_cb.start() + ioloop.start() + +class ChaChaIPToCountryDaemon_Handler(tornado.web.RequestHandler): + def initialize(self, storage): + self.storage = storage + + async def getIP2Country_Full(self,ip): + result = dict() + result["alpha_2"] = await self.getIP2Country(ip) + country = pycountry.countries.get(alpha_2=result["alpha_2"]) + result["coutry_name"]=country.name.upper() + result["alpha_3"]=country.alpha_3 + result["coutry_official_name"]=country.official_name + result["utc_time"]= datetime.now(timezone("UTC")).strftime('%H:%M') + result["ip"]=ip + addr=reversename.from_address(ip) + result["dns"]=str(resolver.query(addr,"PTR")[0])[:-1] + return result + + async def getIP2Country(self,ip): + print("Requested ip address: {0}".format(ip)) + return await self.storage.test_ip(ip) + +class ChaChaIPToCountryDaemon_Handler_REST(ChaChaIPToCountryDaemon_Handler): + async def post(self): + ipToCompute = json.loads(self.request.body) + res = await self.getIP2Country_Full(ipToCompute["ip"]) + self.write(json.dumps(res).encode("utf-8")) + self.finish() + +class ChaChaIPToCountryDaemon_Handler_Legacy(ChaChaIPToCountryDaemon_Handler): + async def get(self): + ipToCompute = self.get_argument('ip') + res = await self.getIP2Country_Full(ipToCompute) + resline= "{0} {1}:{2}:{3}:{4}:{5}".format(res["utc_time"],res["ip"],res["dns"],res["coutry_name"],res["alpha_3"],res["alpha_2"]) + self.write(resline.encode("utf-8")) + self.finish() \ No newline at end of file diff --git a/Tests/TEST_base.py b/Tests/TEST_base.py new file mode 100644 index 0000000..c562331 --- /dev/null +++ b/Tests/TEST_base.py @@ -0,0 +1,20 @@ +from pprint import pprint +from pathlib import Path + +import unittest + +import sys +import io +sys.path.append('../') + +from ChaChaIPToCountryDaemon import * + + +class Test_ChaChaIPToCountryDaemon_base(unittest.TestCase): + + def setUp(self): + [f.unlink() for f in Path("tmp").glob("*")] + print("======================") + + def test_simplerun(self): + tmp = ChaChaIPToCountryDaemon() \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..49b3c4b --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +from setuptools import setup,find_packages +from ChaChaIPToCountryDaemon import __version__ as ChaChaIPToCountryDaemon_Version + +with open("./README.md", "r") as fh: + long_description = fh.read() + +setup( + name='ChaChaIPToCountryDaemon', + version=ChaChaIPToCountryDaemon_Version, + description='An HTTP/REST IP2Country Daemon using country-ip-blocks database.', + author='Clement CHASTANIER', + author_email='clement.chastanier@gmail.com', + long_description=long_description, + long_description_content_type="text/markdown", + url='https://chacha.ddns.net/gitea/chacha/ChaChaIPToCountryDaemon', + packages=find_packages(), + install_requires=["tornado","gitpython","netaddr","aiofiles","pycountry","dnspython","asyncstdlib"], + license="CC BY-NC-SA 4.0", + python_requires='>=3.8', + ), +