Merge pull request #17 from MAVRICK-1/uncompress_ripe_rpki

RIPE RPKI archive format changed (.xz) #14

Author: romain-fontugne

rov/__init__.py

@@ -12,6 +12,8 @@
 import shutil
 import sys
 import csv
+import lzma
+from io import BytesIO
 
 import urllib
 import urllib.request as request
@@ -64,11 +66,12 @@
         'https://rpki.gin.ntt.net/api/export.json'
         ]
 RPKI_ARCHIVE_URLS = [ 
-        'https://ftp.ripe.net/ripe/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
-        'https://ftp.ripe.net/ripe/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
-        'https://ftp.ripe.net/ripe/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
-        'https://ftp.ripe.net/ripe/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
-        'https://ftp.ripe.net/ripe/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv',
+        'https://ftp.ripe.net/rpki/afrinic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
+        'https://ftp.ripe.net/rpki/apnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
+        'https://ftp.ripe.net/rpki/arin.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
+        'https://ftp.ripe.net/rpki/lacnic.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
+        'https://ftp.ripe.net/rpki/ripencc.tal/{year:04d}/{month:02d}/{day:02d}/roas.csv.xz',
+        
         ]
 DEFAULT_DELEGATED_URLS = [ 
         'https://www.nro.net/wp-content/uploads/delegated-stats/nro-extended-stats'
@@ -459,17 +462,25 @@ def download_databases(self, overwrite=True):
 
                 # all files from RIPE's RPKI archive have the same name
                 # 'roas.csv', change it with the tal name
-                if fname == 'roas.csv':
-                    fname = guess_ta_name(url)+'.csv'
+                if fname == 'roas.csv.xz':
+                    fname = guess_ta_name(url)+".csv"
 
                 if os.path.exists(folder+fname) and not overwrite:
                     continue
 
                 sys.stderr.write(f'Downloading: {url}\n')
+        
                 try:
-                    with closing(request.urlopen(url)) as r:
-                        with open(folder+fname, 'wb') as f:
-                            shutil.copyfileobj(r, f)
+                    # to separete csv.xz file to decompress
+                    if "roas.csv.xz" in url:
+                        with closing(request.urlopen(url)) as response:
+                            with lzma.open(BytesIO(response.read())) as r:
+                                with open(folder+fname, 'wb') as f:
+                                    shutil.copyfileobj(r,f)
+                    else:
+                        with closing(request.urlopen(url)) as r:
+                            with open(folder+fname, 'wb') as f:
+                                shutil.copyfileobj(r, f)
                 except urllib.error.URLError:
                     sys.stderr.write(f'Error {url} is not available.\n')
-
+                    

setup.py

@@ -4,7 +4,7 @@
     readme = f.read()
 
 setup(
-    version = '0.4.0',
+    version = '0.5.0',
     name = 'rov',
     author = 'Romain Fontugne',
     author_email = 'romain.fontugne@gmail.com',
@@ -17,7 +17,7 @@
     install_requires=[
         'appdirs',
         'py-radix',
-        'portion',
+        'portion'
     ],
     entry_points={'console_scripts':
             ['rov = rov.__main__:main']},