|
|
|
@ -14,7 +14,7 @@ import datetime |
|
|
|
|
import logging |
|
|
|
|
|
|
|
|
|
from vasl_templates.webapp import app, globvars |
|
|
|
|
from vasl_templates.webapp.utils import parse_int |
|
|
|
|
from vasl_templates.webapp.utils import parse_int, friendly_byte_count |
|
|
|
|
|
|
|
|
|
_registry = set() |
|
|
|
|
_logger = logging.getLogger( "downloads" ) |
|
|
|
@ -90,7 +90,7 @@ class DownloadedFile: |
|
|
|
|
self._lock.release() |
|
|
|
|
|
|
|
|
|
@staticmethod |
|
|
|
|
def download_files(): |
|
|
|
|
def download_files(): #pylint: disable=too-many-locals |
|
|
|
|
"""Download fresh copies of each file.""" |
|
|
|
|
#pylint: disable=protected-access |
|
|
|
|
|
|
|
|
@ -142,10 +142,15 @@ class DownloadedFile: |
|
|
|
|
with urllib.request.urlopen( req ) as resp: |
|
|
|
|
resp_data = resp.read() |
|
|
|
|
if resp.headers.get( "Content-Encoding" ) == "gzip": |
|
|
|
|
gzip_byte_count_str = " ({})".format( friendly_byte_count( len(resp_data) ) ) |
|
|
|
|
resp_data = gzip.decompress( resp_data ) |
|
|
|
|
else: |
|
|
|
|
gzip_byte_count_str = "" |
|
|
|
|
data = resp_data.decode( "utf-8" ) |
|
|
|
|
etag = resp.headers.get( "ETag" ) |
|
|
|
|
_logger.info( "Downloaded the %s file OK: %d bytes", df.key, len(data) ) |
|
|
|
|
_logger.info( "Downloaded the %s file OK: %s", df.key, |
|
|
|
|
friendly_byte_count( len(data) ) + gzip_byte_count_str |
|
|
|
|
) |
|
|
|
|
if etag: |
|
|
|
|
_logger.debug( "- Got etag: %s", etag ) |
|
|
|
|
_etags[ url ] = etag |
|
|
|
|