DEV
-continue implementation
This commit is contained in:
cclecle
17
.project
Normal file
17
.project
Normal file
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>ChaChaIPToCountryDaemon</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.python.pydev.PyDevBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.python.pydev.pythonNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
13
ChaChaIPToCountryDaemon/__init__.py
Normal file
13
ChaChaIPToCountryDaemon/__init__.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
ChaChaIPToCountryDaemon (c) by clement chastanier
|
||||
|
||||
ChaChaIPToCountryDaemon is licensed under a
|
||||
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
|
||||
|
||||
You should have received a copy of the license along with this
|
||||
work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
|
||||
"""
|
||||
|
||||
from ._version import __version__
|
||||
|
||||
from .core import ChaChaIPToCountryDaemon
|
||||
11
ChaChaIPToCountryDaemon/_version.py
Normal file
11
ChaChaIPToCountryDaemon/_version.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
ChaChaIPToCountryDaemon (c) by clement chastanier
|
||||
|
||||
ChaChaIPToCountryDaemon is licensed under a
|
||||
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
|
||||
|
||||
You should have received a copy of the license along with this
|
||||
work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
|
||||
"""
|
||||
|
||||
__version__ = "0.1"
|
||||
215
ChaChaIPToCountryDaemon/core.py
Normal file
215
ChaChaIPToCountryDaemon/core.py
Normal file
@@ -0,0 +1,215 @@
|
||||
"""
|
||||
ChaChaIPToCountryDaemon (c) by clement chastanier
|
||||
|
||||
ChaChaIPToCountryDaemon is licensed under a
|
||||
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
|
||||
|
||||
You should have received a copy of the license along with this
|
||||
work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
|
||||
"""
|
||||
|
||||
|
||||
from typing import Union
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
from tornado.httpclient import AsyncHTTPClient
|
||||
import json
|
||||
import tempfile
|
||||
from git import Repo
|
||||
import os
|
||||
from pathlib import Path
|
||||
from netaddr import IPSet,IPAddress
|
||||
import asyncio
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from ipaddress import ip_address, IPv4Address
|
||||
import pycountry
|
||||
from datetime import datetime, timedelta
|
||||
from pytz import timezone
|
||||
from functools import wraps
|
||||
from asyncstdlib import lru_cache
|
||||
from dns import resolver,reversename
|
||||
|
||||
DEFAULT__Cache_Expiration_Second = 600
|
||||
DEFAULT__Cache_Size_Entries = 2048
|
||||
DEFAULT__IpDataSet_UpdatePeriod_Second = 120
|
||||
DEFAULT__Num_IpDataSet_Workers = 10
|
||||
DEFAULT__IpDataSet_GitRepo_Address = "https://chacha.ddns.net/gitea/chacha/country-ip-blocks"
|
||||
DEFAULT__IpDataSet_ipv4_subdir = "ipv4"
|
||||
DEFAULT__IpDataSet_ipv6_subdir = "ipv6"
|
||||
|
||||
def processfile(_entry_path,_entry_code) :
|
||||
with open(_entry_path) as fp:
|
||||
_set = IPSet()
|
||||
for line in fp:
|
||||
_set.add(line)
|
||||
print("PARSE DONE({0})".format(_entry_path))
|
||||
return _entry_code,_set
|
||||
|
||||
def validIPAddress(IP: str) -> str:
|
||||
try:
|
||||
return "IPv4" if type(ip_address(IP)) is IPv4Address else "IPv6"
|
||||
except ValueError:
|
||||
return "Invalid"
|
||||
|
||||
def timed_lru_cache(seconds: int, maxsize: int = None):
|
||||
def wrapper_cache(func):
|
||||
func = lru_cache(maxsize=maxsize)(func)
|
||||
func.lifetime = timedelta(seconds=seconds)
|
||||
func.expiration = datetime.utcnow() + func.lifetime
|
||||
@wraps(func)
|
||||
def wrapped_func(*args, **kwargs):
|
||||
if datetime.utcnow() >= func.expiration:
|
||||
func.cache_clear()
|
||||
func.expiration = datetime.utcnow() + func.lifetime
|
||||
return func(*args, **kwargs)
|
||||
return wrapped_func
|
||||
return wrapper_cache
|
||||
|
||||
class ChaChaIPToCountryStorage:
|
||||
def __init__(self):
|
||||
self.ipv4set = dict()
|
||||
self.ipv6set = dict()
|
||||
self.tempdir = tempfile.TemporaryDirectory()
|
||||
self.tempdir_url = self.tempdir.name + os.sep
|
||||
self.ipv4_dir = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv4_subdir)
|
||||
self.ipv6_dir = os.path.join(self.tempdir.name,DEFAULT__IpDataSet_ipv6_subdir)
|
||||
self.DataSetlock = asyncio.Lock()
|
||||
print("New dataset dir = " + self.tempdir_url)
|
||||
print("Cloning GIT repos: "+ DEFAULT__IpDataSet_GitRepo_Address)
|
||||
self.gitrepo = Repo.clone_from(DEFAULT__IpDataSet_GitRepo_Address,self.tempdir_url )
|
||||
print("Done")
|
||||
asyncio.get_event_loop().run_until_complete(self.update_repo())
|
||||
print("Finished")
|
||||
|
||||
@timed_lru_cache(seconds=DEFAULT__Cache_Expiration_Second,maxsize=DEFAULT__Cache_Size_Entries)
|
||||
async def test_ip(self,ip):
|
||||
validity = validIPAddress(ip)
|
||||
|
||||
if validity == "IPv4":
|
||||
set =self.ipv4set
|
||||
elif validity == "IPv6":
|
||||
set =self.ipv6set
|
||||
else:
|
||||
RuntimeError("Invalid ip address received")
|
||||
|
||||
await self.DataSetlock.acquire()
|
||||
try:
|
||||
for key,set in set.items():
|
||||
if IPAddress(ip) in set:
|
||||
return key
|
||||
finally:
|
||||
self.DataSetlock.release()
|
||||
|
||||
return "ZZ"
|
||||
|
||||
async def update_repo(self):
|
||||
print("== DATASET UPDATE REQUESTED ==")
|
||||
print("Pulling GIT dataset repo")
|
||||
o = self.gitrepo.remotes.origin
|
||||
o.pull()
|
||||
print("Done")
|
||||
|
||||
tasksIPV4 = []
|
||||
tasksIPV6 = []
|
||||
loop = asyncio.get_event_loop()
|
||||
_executor = ProcessPoolExecutor(DEFAULT__Num_IpDataSet_Workers)
|
||||
|
||||
print("Parsing IPV4 dataset")
|
||||
with os.scandir(self.ipv4_dir) as entries:
|
||||
for entry in entries:
|
||||
entry_path = os.path.join(self.ipv4_dir,entry.name)
|
||||
entry_code = Path(entry_path).stem
|
||||
if entry_code not in ["fr"]: continue
|
||||
task = loop.run_in_executor(_executor,processfile,entry_path,entry_code)
|
||||
tasksIPV4.append(task)
|
||||
print("Done")
|
||||
|
||||
#print("Parsing IPV6 dataset")
|
||||
#with os.scandir(self.ipv6_dir) as entries:
|
||||
# for entry in entries:
|
||||
# entry_path = os.path.join(self.ipv6_dir,entry.name)
|
||||
# entry_code = Path(entry_path).stem
|
||||
# if entry_code not in ["fr"]: continue
|
||||
# task = loop.run_in_executor(_executor,processfile,entry_path,entry_code)
|
||||
# tasksIPV6.append(task)
|
||||
#print("Done")
|
||||
|
||||
print("Wait IPV4 parsing to complete...")
|
||||
results = await asyncio.gather(*tasksIPV4)
|
||||
_ipv4set=dict()
|
||||
for _entry_code,_set in results:
|
||||
_ipv4set[_entry_code] = _set
|
||||
print("Done")
|
||||
|
||||
print("Updating live IPV4 storage")
|
||||
await self.DataSetlock.acquire()
|
||||
try:
|
||||
self.ipv4set = _ipv4set
|
||||
finally:
|
||||
self.DataSetlock.release()
|
||||
print("Done")
|
||||
|
||||
print("Wait IPV6 parsing to complete...")
|
||||
results = await asyncio.gather(*tasksIPV6)
|
||||
_ipv6set=dict()
|
||||
for _entry_code,_set in results:
|
||||
_ipv6set[_entry_code] = _set
|
||||
print("Done")
|
||||
|
||||
print("Updating live IPV6 storage")
|
||||
await self.DataSetlock.acquire()
|
||||
try:
|
||||
self.ipv6set = _ipv6set
|
||||
finally:
|
||||
self.DataSetlock.release()
|
||||
print("Done")
|
||||
print("== DATASET UPDATE FINISHED ==")
|
||||
|
||||
class ChaChaIPToCountryDaemon:
|
||||
def __init__(self,serverport:int=80,baseurl:str=""):
|
||||
self.storage = ChaChaIPToCountryStorage()
|
||||
self.handler = tornado.web.Application([
|
||||
(baseurl + r"/api/ip2country/", ChaChaIPToCountryDaemon_Handler_REST, {'storage':self.storage}), # new RESTfull API handler (POST)
|
||||
(baseurl + r"/ip2country.php", ChaChaIPToCountryDaemon_Handler_Legacy, {'storage':self.storage}), # legacy php-like handler (GET)
|
||||
])
|
||||
self.handler.listen(serverport)
|
||||
ioloop = tornado.ioloop.IOLoop.current()
|
||||
update_cb = tornado.ioloop.PeriodicCallback(self.storage.update_repo, 1000*DEFAULT__IpDataSet_UpdatePeriod_Second)
|
||||
update_cb.start()
|
||||
ioloop.start()
|
||||
|
||||
class ChaChaIPToCountryDaemon_Handler(tornado.web.RequestHandler):
|
||||
def initialize(self, storage):
|
||||
self.storage = storage
|
||||
|
||||
async def getIP2Country_Full(self,ip):
|
||||
result = dict()
|
||||
result["alpha_2"] = await self.getIP2Country(ip)
|
||||
country = pycountry.countries.get(alpha_2=result["alpha_2"])
|
||||
result["coutry_name"]=country.name.upper()
|
||||
result["alpha_3"]=country.alpha_3
|
||||
result["coutry_official_name"]=country.official_name
|
||||
result["utc_time"]= datetime.now(timezone("UTC")).strftime('%H:%M')
|
||||
result["ip"]=ip
|
||||
addr=reversename.from_address(ip)
|
||||
result["dns"]=str(resolver.query(addr,"PTR")[0])[:-1]
|
||||
return result
|
||||
|
||||
async def getIP2Country(self,ip):
|
||||
print("Requested ip address: {0}".format(ip))
|
||||
return await self.storage.test_ip(ip)
|
||||
|
||||
class ChaChaIPToCountryDaemon_Handler_REST(ChaChaIPToCountryDaemon_Handler):
|
||||
async def post(self):
|
||||
ipToCompute = json.loads(self.request.body)
|
||||
res = await self.getIP2Country_Full(ipToCompute["ip"])
|
||||
self.write(json.dumps(res).encode("utf-8"))
|
||||
self.finish()
|
||||
|
||||
class ChaChaIPToCountryDaemon_Handler_Legacy(ChaChaIPToCountryDaemon_Handler):
|
||||
async def get(self):
|
||||
ipToCompute = self.get_argument('ip')
|
||||
res = await self.getIP2Country_Full(ipToCompute)
|
||||
resline= "{0} {1}:{2}:{3}:{4}:{5}".format(res["utc_time"],res["ip"],res["dns"],res["coutry_name"],res["alpha_3"],res["alpha_2"])
|
||||
self.write(resline.encode("utf-8"))
|
||||
self.finish()
|
||||
20
Tests/TEST_base.py
Normal file
20
Tests/TEST_base.py
Normal file
@@ -0,0 +1,20 @@
|
||||
from pprint import pprint
|
||||
from pathlib import Path
|
||||
|
||||
import unittest
|
||||
|
||||
import sys
|
||||
import io
|
||||
sys.path.append('../')
|
||||
|
||||
from ChaChaIPToCountryDaemon import *
|
||||
|
||||
|
||||
class Test_ChaChaIPToCountryDaemon_base(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
[f.unlink() for f in Path("tmp").glob("*")]
|
||||
print("======================")
|
||||
|
||||
def test_simplerun(self):
|
||||
tmp = ChaChaIPToCountryDaemon()
|
||||
23
setup.py
Normal file
23
setup.py
Normal file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from setuptools import setup,find_packages
|
||||
from ChaChaIPToCountryDaemon import __version__ as ChaChaIPToCountryDaemon_Version
|
||||
|
||||
with open("./README.md", "r") as fh:
|
||||
long_description = fh.read()
|
||||
|
||||
setup(
|
||||
name='ChaChaIPToCountryDaemon',
|
||||
version=ChaChaIPToCountryDaemon_Version,
|
||||
description='An HTTP/REST IP2Country Daemon using country-ip-blocks database.',
|
||||
author='Clement CHASTANIER',
|
||||
author_email='clement.chastanier@gmail.com',
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
url='https://chacha.ddns.net/gitea/chacha/ChaChaIPToCountryDaemon',
|
||||
packages=find_packages(),
|
||||
install_requires=["tornado","gitpython","netaddr","aiofiles","pycountry","dnspython","asyncstdlib"],
|
||||
license="CC BY-NC-SA 4.0",
|
||||
python_requires='>=3.8',
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user