From 427a3d65c575111e133e882ec6ed9e031b5d68a4 Mon Sep 17 00:00:00 2001 From: Taka Date: Sat, 2 Oct 2021 05:29:15 +1000 Subject: [PATCH] Updated all dependencies. --- .pylintrc | 4 +- conftest.py | 14 +- freeze.py | 318 +++++++++--------- loader/main.py | 5 +- requirements-dev.txt | 12 +- requirements.txt | 8 +- setup.py | 15 +- tools/build_file.py | 4 +- vasl_templates/main.py | 15 +- vasl_templates/tools/check_connect_roar.py | 6 +- .../tools/make_chapter_h_placeholders.py | 19 +- .../tests/test_make_chapter_h_placeholders.py | 3 +- vasl_templates/tools/webdriver_stress_test.py | 6 +- vasl_templates/utils.py | 3 +- vasl_templates/webapp/__init__.py | 10 +- vasl_templates/webapp/downloads.py | 12 +- vasl_templates/webapp/files.py | 8 +- vasl_templates/webapp/main.py | 4 +- vasl_templates/webapp/run_server.py | 3 +- vasl_templates/webapp/scenarios.py | 12 +- vasl_templates/webapp/snippets.py | 7 +- vasl_templates/webapp/static/help/index.html | 2 +- .../webapp/tests/control_tests_servicer.py | 2 +- vasl_templates/webapp/tests/test_counters.py | 21 +- vasl_templates/webapp/tests/test_files.py | 46 +-- vasl_templates/webapp/tests/test_lfa.py | 3 +- .../tests/test_national_capabilities.py | 15 +- .../webapp/tests/test_scenario_persistence.py | 4 +- .../webapp/tests/test_scenario_search.py | 3 +- .../webapp/tests/test_template_packs.py | 57 ++-- .../webapp/tests/test_vasl_extensions.py | 3 +- vasl_templates/webapp/tests/test_vassal.py | 6 +- .../webapp/tests/test_vehicles_ordnance.py | 6 +- vasl_templates/webapp/tests/test_vo_notes.py | 37 +- .../webapp/tests/test_vo_reports.py | 15 +- vasl_templates/webapp/tests/utils.py | 3 +- vasl_templates/webapp/utils.py | 2 +- vasl_templates/webapp/vasl_mod.py | 15 +- vasl_templates/webapp/vassal.py | 18 +- vasl_templates/webapp/vo_notes.py | 23 +- vasl_templates/webapp/vo_utils.py | 3 +- vasl_templates/webapp/webdriver.py | 2 +- 42 files changed, 412 insertions(+), 362 deletions(-) diff --git a/.pylintrc b/.pylintrc index 0678224..95add7c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -146,7 +146,9 @@ disable=print-statement, len-as-condition, consider-using-enumerate, import-outside-toplevel, - isinstance-second-argument-not-valid-type + isinstance-second-argument-not-valid-type, + consider-using-f-string, + consider-using-max-builtin # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/conftest.py b/conftest.py index 2587538..08c4941 100644 --- a/conftest.py +++ b/conftest.py @@ -136,7 +136,7 @@ def _make_webapp(): # This means that the webapp doesn't get a chance to shutdown properly (in particular, # clean up the gRPC service), but since we send an EndTests message at the of each test, # the remote server gets a chance to clean up then. It's not perfect (e.g. if the tests fail - # or otherwise finish eearly before they get a chance to send the EndTests message), but + # or otherwise finish early before they get a chance to send the EndTests message), but # we can live with it. thread = threading.Thread( target = lambda: app.run( host="0.0.0.0", port=FLASK_WEBAPP_PORT, use_reloader=False ), @@ -147,8 +147,8 @@ def _make_webapp(): def is_ready(): """Try to connect to the webapp server.""" try: - resp = urllib.request.urlopen( app.url_for("ping") ).read() - assert resp.startswith( b"pong: " ) + with urllib.request.urlopen( app.url_for("ping") ) as resp: + assert resp.read().startswith( b"pong: " ) return True except URLError: return False @@ -158,14 +158,14 @@ def _make_webapp(): # set up control of the remote webapp server try: - resp = json.load( - urllib.request.urlopen( app.url_for( "get_control_tests" ) ) - ) + url = app.url_for( "get_control_tests" ) + with urllib.request.urlopen( url ) as resp: + resp_data = json.load( resp ) except urllib.error.HTTPError as ex: if ex.code == 404: raise RuntimeError( "Can't get the test control port - has remote test control been enabled?" ) from ex raise - port_no = resp.get( "port" ) + port_no = resp_data.get( "port" ) if not port_no: raise RuntimeError( "The webapp server is not running the test control service." ) mo = re.search( r"^http://(.+):\d+$", app.base_url ) diff --git a/freeze.py b/freeze.py index 97e8618..15ff9b1 100755 --- a/freeze.py +++ b/freeze.py @@ -21,6 +21,168 @@ APP_ICON = os.path.join( BASE_DIR, "vasl_templates/webapp/static/images/app.ico" # --------------------------------------------------------------------- +def main( args ): #pylint: disable=too-many-locals + """Main processing.""" + + # parse the command-line options + output_fname = None + no_loader = False + work_dir = None + cleanup = True + opts,args = getopt.getopt( sys.argv[1:], "o:w:", ["output=","no-loader","work=","no-clean"] ) + for opt, val in opts: + if opt in ["-o","--output"]: + output_fname = val.strip() + elif opt in ["--no-loader"]: + no_loader = True + elif opt in ["-w","--work"]: + work_dir = val.strip() + elif opt in ["--no-clean"]: + cleanup = False + else: + raise RuntimeError( "Unknown argument: {}".format( opt ) ) + if not output_fname: + raise RuntimeError( "No output file was specified." ) + + # figure out where to locate our work directories + if work_dir: + work_dir = os.path.abspath( work_dir ) + build_dir = os.path.join( work_dir, "build" ) + if os.path.isdir( build_dir ): + shutil.rmtree( build_dir ) + dist_dir = os.path.join( work_dir, "dist" ) + if os.path.isdir( dist_dir ): + shutil.rmtree( dist_dir ) + else: + build_dir = tempfile.mkdtemp() + dist_dir = tempfile.mkdtemp() + + # figure out the format of the release archive + formats = { ".zip": "zip", ".tar.gz": "gztar", ".tar.bz": "bztar", ".tar": "tar" } + output_fmt = None + for extn,fmt in formats.items(): + if output_fname.endswith( extn ): + output_fmt = fmt + output_fname2 = output_fname[:-len(extn)] + break + if not output_fmt: + raise RuntimeError( "Unknown release archive format: {}".format( os.path.split(output_fname)[1] ) ) + + # configure pyinstaller + # NOTE: Using UPX gave ~25% saving on Windows, but failed to run because of corrupt DLL's :-/ + target_name = make_target_name( "vasl-templates" ) + args = [ + "--distpath", dist_dir, + "--workpath", build_dir, + "--specpath", build_dir, + "--onefile", + "--name", target_name, + ] + args.extend( [ "--add-data", + os.path.join( BASE_DIR, "vassal-shim/release/vassal-shim.jar" + os.pathsep + "vasl_templates/webapp" ) + ] ) + # NOTE: We also need to include the config/ and data/ subdirectories, but we would like to + # make them available to the user, so we include them ourself in the final release archive. + def map_dir( src, dest ): #pylint: disable=missing-docstring + args.extend( [ "--add-data", + os.path.join( BASE_DIR, src + os.pathsep + dest ) + ] ) + map_dir( "vasl_templates/ui", "vasl_templates/ui" ) + map_dir( "vasl_templates/resources", "vasl_templates/resources" ) + map_dir( "vasl_templates/webapp/static", "vasl_templates/webapp/static" ) + map_dir( "vasl_templates/webapp/templates", "vasl_templates/webapp/templates" ) + if sys.platform == "win32": + args.append( "--noconsole" ) + args.extend( [ "--icon", APP_ICON ] ) + # NOTE: These files are not always required but it's probably safer to always include them. + import distutils.sysconfig #pylint: disable=import-error + dname = os.path.join( distutils.sysconfig.get_python_lib() , "PyQt5/Qt5/bin" ) + args.extend( [ "--add-binary", os.path.join(dname,"libEGL.dll") + os.pathsep + "PyQt5/Qt/bin" ] ) + args.extend( [ "--add-binary", os.path.join(dname,"libGLESv2.dll") + os.pathsep + "PyQt5/Qt/bin" ] ) + args.append( MAIN_SCRIPT ) + + # freeze the application + start_time = time.time() + os.chdir( BASE_DIR ) + run_pyinstaller( args ) # nb: this doesn't return any indication if it worked or not :-/ + + # add extra files to the distribution + def ignore_files( dname, fnames ): #pylint: disable=redefined-outer-name + """Return files to ignore during copytree().""" + # ignore cache files + ignore = [ "__pycache__", "GPUCache" ] + # ignore dot files + ignore.extend( f for f in fnames if f.startswith(".") ) + # ignore Python files + ignore.extend( f for f in fnames if os.path.splitext(f)[1] == ".py" ) + # ignore anything in .gitignore + fname = os.path.join( dname, ".gitignore" ) + if os.path.isfile( fname ): + with open( fname, "r", encoding="utf-8" ) as fp: + for line_buf in fp: + line_buf = line_buf.strip() + if not line_buf or line_buf.startswith("#"): + continue + ignore.append( line_buf ) # nb: we assume normal filenames i.e. no globbing + return ignore + shutil.copy( "LICENSE.txt", dist_dir ) + shutil.copytree( "vasl_templates/webapp/data", os.path.join(dist_dir,"data") ) + shutil.copytree( "vasl_templates/webapp/config", os.path.join(dist_dir,"config"), ignore=ignore_files ) + + # copy the examples + dname = os.path.join( dist_dir, "examples" ) + os.makedirs( dname ) + fnames = [ f for f in os.listdir("examples") if os.path.splitext(f)[1] in (".json",".png") ] + for f in fnames: + shutil.copy( os.path.join("examples",f), dname ) + + # set the build info + build_info = { + "timestamp": int( time.time() ), + } + build_info.update( get_git_info() ) + dname = os.path.join( dist_dir, "config" ) + fname = os.path.join( dname, "build-info.json" ) + with open( fname, "w", encoding="utf-8" ) as fp: + json.dump( build_info, fp ) + + # freeze the loader + if no_loader: + print( "Not including the loader." ) + else: + print( "--- BEGIN FREEZE LOADER ---" ) + shutil.move( + os.path.join( dist_dir, target_name ), + os.path.join( dist_dir, make_target_name("vasl-templates-main") ) + ) + from loader.freeze import freeze_loader #pylint: disable=no-name-in-module + freeze_loader( + os.path.join( dist_dir, target_name ), + build_dir, # nb: a "loader" sub-directory will be created and used + False # nb: we will clean up, or not, everything ourself + ) + + # create the release archive + os.chdir( dist_dir ) + print() + print( "Generating release archive: {}".format( output_fname ) ) + shutil.make_archive( output_fname2, output_fmt ) + file_size = os.path.getsize( output_fname ) + print( "- Done: {0:.1f} MB".format( float(file_size) / 1024 / 1024 ) ) + + # clean up + if cleanup: + os.chdir( BASE_DIR ) # so we can delete the build directory :-/ + shutil.rmtree( build_dir ) + shutil.rmtree( dist_dir ) + + # log the elapsed time + elapsed_time = time.time() - start_time + print() + print( "Elapsed time: {}".format( datetime.timedelta( seconds=int(elapsed_time) ) ) ) + +# --------------------------------------------------------------------- + def get_git_info(): """Get the git branch/commit we're building from.""" @@ -55,157 +217,5 @@ def make_target_name( fname ): # --------------------------------------------------------------------- -# parse the command-line options -output_fname = None -no_loader = False -work_dir = None -cleanup = True -opts,args = getopt.getopt( sys.argv[1:], "o:w:", ["output=","no-loader","work=","no-clean"] ) -for opt, val in opts: - if opt in ["-o","--output"]: - output_fname = val.strip() - elif opt in ["--no-loader"]: - no_loader = True - elif opt in ["-w","--work"]: - work_dir = val.strip() - elif opt in ["--no-clean"]: - cleanup = False - else: - raise RuntimeError( "Unknown argument: {}".format( opt ) ) -if not output_fname: - raise RuntimeError( "No output file was specified." ) - -# figure out where to locate our work directories -if work_dir: - work_dir = os.path.abspath( work_dir ) - build_dir = os.path.join( work_dir, "build" ) - if os.path.isdir( build_dir ): - shutil.rmtree( build_dir ) - dist_dir = os.path.join( work_dir, "dist" ) - if os.path.isdir( dist_dir ): - shutil.rmtree( dist_dir ) -else: - build_dir = tempfile.mkdtemp() - dist_dir = tempfile.mkdtemp() - -# figure out the format of the release archive -formats = { ".zip": "zip", ".tar.gz": "gztar", ".tar.bz": "bztar", ".tar": "tar" } -output_fmt = None -for extn,fmt in formats.items(): - if output_fname.endswith( extn ): - output_fmt = fmt - output_fname2 = output_fname[:-len(extn)] - break -if not output_fmt: - raise RuntimeError( "Unknown release archive format: {}".format( os.path.split(output_fname)[1] ) ) - -# configure pyinstaller -# NOTE: Using UPX gave ~25% saving on Windows, but failed to run because of corrupt DLL's :-/ -target_name = make_target_name( "vasl-templates" ) -args = [ - "--distpath", dist_dir, - "--workpath", build_dir, - "--specpath", build_dir, - "--onefile", - "--name", target_name, -] -args.extend( [ "--add-data", - os.path.join( BASE_DIR, "vassal-shim/release/vassal-shim.jar" + os.pathsep + "vasl_templates/webapp" ) -] ) -# NOTE: We also need to include the config/ and data/ subdirectories, but we would like to -# make them available to the user, so we include them ourself in the final release archive. -def map_dir( src, dest ): #pylint: disable=missing-docstring - args.extend( [ "--add-data", - os.path.join( BASE_DIR, src + os.pathsep + dest ) - ] ) -map_dir( "vasl_templates/ui", "vasl_templates/ui" ) -map_dir( "vasl_templates/resources", "vasl_templates/resources" ) -map_dir( "vasl_templates/webapp/static", "vasl_templates/webapp/static" ) -map_dir( "vasl_templates/webapp/templates", "vasl_templates/webapp/templates" ) -if sys.platform == "win32": - args.append( "--noconsole" ) - args.extend( [ "--icon", APP_ICON ] ) - # NOTE: These files are not always required but it's probably safer to always include them. - import distutils.sysconfig #pylint: disable=import-error - dname = os.path.join( distutils.sysconfig.get_python_lib() , "PyQt5/Qt/bin" ) - args.extend( [ "--add-binary", os.path.join(dname,"libEGL.dll") + os.pathsep + "PyQt5/Qt/bin" ] ) - args.extend( [ "--add-binary", os.path.join(dname,"libGLESv2.dll") + os.pathsep + "PyQt5/Qt/bin" ] ) -args.append( MAIN_SCRIPT ) - -# freeze the application -start_time = time.time() -os.chdir( BASE_DIR ) -run_pyinstaller( args ) # nb: this doesn't return any indication if it worked or not :-/ - -# add extra files to the distribution -def ignore_files( dname, fnames ): #pylint: disable=redefined-outer-name - """Return files to ignore during copytree().""" - # ignore cache files - ignore = [ "__pycache__", "GPUCache" ] - # ignore dot files - ignore.extend( f for f in fnames if f.startswith(".") ) - # ignore Python files - ignore.extend( f for f in fnames if os.path.splitext(f)[1] == ".py" ) - # ignore anything in .gitignore - fname = os.path.join( dname, ".gitignore" ) - if os.path.isfile( fname ): - for line_buf in open(fname,"r"): - line_buf = line_buf.strip() - if not line_buf or line_buf.startswith("#"): - continue - ignore.append( line_buf ) # nb: we assume normal filenames i.e. no globbing - return ignore -shutil.copy( "LICENSE.txt", dist_dir ) -shutil.copytree( "vasl_templates/webapp/data", os.path.join(dist_dir,"data") ) -shutil.copytree( "vasl_templates/webapp/config", os.path.join(dist_dir,"config"), ignore=ignore_files ) - -# copy the examples -dname = os.path.join( dist_dir, "examples" ) -os.makedirs( dname ) -fnames = [ f for f in os.listdir("examples") if os.path.splitext(f)[1] in (".json",".png") ] -for f in fnames: - shutil.copy( os.path.join("examples",f), dname ) - -# set the build info -build_info = { - "timestamp": int( time.time() ), -} -build_info.update( get_git_info() ) -dname = os.path.join( dist_dir, "config" ) -with open( os.path.join(dname,"build-info.json"), "w" ) as fp: - json.dump( build_info, fp ) - -# freeze the loader -if no_loader: - print( "Not including the loader." ) -else: - print( "--- BEGIN FREEZE LOADER ---" ) - shutil.move( - os.path.join( dist_dir, target_name ), - os.path.join( dist_dir, make_target_name("vasl-templates-main") ) - ) - from loader.freeze import freeze_loader #pylint: disable=no-name-in-module - freeze_loader( - os.path.join( dist_dir, target_name ), - build_dir, # nb: a "loader" sub-directory will be created and used - False # nb: we will clean up, or not, everything ourself - ) - -# create the release archive -os.chdir( dist_dir ) -print() -print( "Generating release archive: {}".format( output_fname ) ) -shutil.make_archive( output_fname2, output_fmt ) -file_size = os.path.getsize( output_fname ) -print( "- Done: {0:.1f} MB".format( float(file_size) / 1024 / 1024 ) ) - -# clean up -if cleanup: - os.chdir( BASE_DIR ) # so we can delete the build directory :-/ - shutil.rmtree( build_dir ) - shutil.rmtree( dist_dir ) - -# log the elapsed time -elapsed_time = time.time() - start_time -print() -print( "Elapsed time: {}".format( datetime.timedelta( seconds=int(elapsed_time) ) ) ) +if __name__ == "__main__": + main( sys.argv[1:] ) diff --git a/loader/main.py b/loader/main.py index f5c6090..ac9fedd 100755 --- a/loader/main.py +++ b/loader/main.py @@ -58,7 +58,7 @@ def main( args ): # launch the main vasl-templates program try: - proc = subprocess.Popen( itertools.chain( [fname], args ) ) + proc = subprocess.Popen( itertools.chain( [fname], args ) ) #pylint: disable=consider-using-with except Exception as ex: #pylint: disable=broad-except show_error_msg( "Can't start vasl-templates:\n\n{}".format( ex ), withdraw=True ) return -2 @@ -146,7 +146,8 @@ def check_startup( proc, port ): # check if the webapp is responding url = "http://localhost:{}/ping".format( port ) try: - _ = urllib.request.urlopen( url ).read() + with urllib.request.urlopen( url ) as resp: + _ = resp.read() except URLError: # no response - the webapp is probably still starting up return False diff --git a/requirements-dev.txt b/requirements-dev.txt index 3251b80..2edcd3a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ -pytest==6.2.1 -grpcio-tools==1.34.1 -tabulate==0.8.7 -lxml==4.6.2 -pylint==2.6.0 +pytest==6.2.5 +grpcio-tools==1.41.0 +tabulate==0.8.9 +lxml==4.6.3 +pylint==2.11.1 pytest-pylint==0.18.0 -pyinstaller==4.2 +pyinstaller==4.5.1 diff --git a/requirements.txt b/requirements.txt index fc0955d..776e3a7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # python 3.8.7 -flask==1.1.2 -pyyaml==5.3.1 -pillow==8.1.0 +flask==2.0.1 +pyyaml==5.4.1 +pillow==8.3.2 selenium==3.141.0 -click==7.1.2 +click==8.0.1 diff --git a/setup.py b/setup.py index 7651326..8f2ef29 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,12 @@ def parse_requirements( fname ): """Parse a requirements file.""" lines = [] fname = os.path.join( os.path.split(__file__)[0], fname ) - for line in open(fname,"r"): - line = line.strip() - if line == "" or line.startswith("#"): - continue - lines.append( line ) + with open( fname, "r", encoding="utf-8" ) as fp: + for line in fp: + line = line.strip() + if line == "" or line.startswith("#"): + continue + lines.append( line ) return lines # --------------------------------------------------------------------- @@ -38,8 +39,8 @@ setup( # NOTE: PyQt5 requirements: https://doc.qt.io/qt-5/linux.html # Linux: mesa-libGL-devel ; @"C Development Tools and Libraries" # NOTE: You may need to disable VMware 3D acceleration, if QWebEngineView is crashing. - "PyQT5==5.15.2", - "PyQtWebEngine==5.15.2", + "PyQT5==5.15.4", + "PyQtWebEngine==5.15.4", ], "dev": parse_requirements( "requirements-dev.txt" ), }, diff --git a/tools/build_file.py b/tools/build_file.py index 423fef0..bbe2978 100755 --- a/tools/build_file.py +++ b/tools/build_file.py @@ -173,8 +173,8 @@ def main( input_file, line_nos, images ): # check if we've been given a .vmod file if os.path.splitext( input_file.name )[1] == ".vmod": # yup - extract the build file - zip_file = zipfile.ZipFile( input_file.name, "r" ) - build_file = zip_file.read( "buildFile" ) + with zipfile.ZipFile( input_file.name, "r" ) as zf: + build_file = zf.read( "buildFile" ) else: # nope - read the build file from the specified file build_file = input_file.read() diff --git a/vasl_templates/main.py b/vasl_templates/main.py index 453f096..649dd33 100755 --- a/vasl_templates/main.py +++ b/vasl_templates/main.py @@ -76,7 +76,7 @@ def main( template_pack, default_scenario, remote_debugging, debug ): # assume too much about how much of our expected environment has been set up. try: fname = os.path.join( QDir.homePath(), "vasl-templates.log" ) - with open( fname, "w" ) as fp: + with open( fname, "w", encoding="utf-8" ) as fp: traceback.print_exc( file=fp ) except: #pylint: disable=bare-except pass @@ -173,12 +173,13 @@ def _do_main( template_pack, default_scenario, remote_debugging, debug ): #pylin break try: url = "http://localhost:{}/ping".format( port ) - resp = urllib.request.urlopen( url ).read().decode( "utf-8" ) - # we got a response - figure out if we connected to ourself or another instance - if resp[:6] != "pong: ": - raise SimpleError( "Unexpected server check response: {}".format( resp ) ) - if resp[6:] == INSTANCE_ID: - break + with urllib.request.urlopen( url ) as resp: + resp_data = resp.read().decode( "utf-8" ) + # we got a response - figure out if we connected to ourself or another instance + if resp_data[:6] != "pong: ": + raise SimpleError( "Unexpected server check response: {}".format( resp_data ) ) + if resp_data[6:] == INSTANCE_ID: + break from vasl_templates.webapp.config.constants import APP_NAME QMessageBox.warning( None, APP_NAME, "The program is already running." ) return -1 diff --git a/vasl_templates/tools/check_connect_roar.py b/vasl_templates/tools/check_connect_roar.py index cb0ad52..629a7bf 100755 --- a/vasl_templates/tools/check_connect_roar.py +++ b/vasl_templates/tools/check_connect_roar.py @@ -25,12 +25,14 @@ def roar_string( s ): # load the ASL Scenario Archive scenarios fname = sys.argv[1] -asa_data = json.load( open( fname, "r" ) ) +with open( fname, "r", encoding="utf-8" ) as fp: + asa_data = json.load( fp ) _build_asa_scenario_index( _asa_scenarios, asa_data, None ) # load the ROAR scenarios fname = sys.argv[2] -roar_data = json.load( open( fname, "r" ) ) +with open( fname, "r", encoding="utf-8" ) as fp: + roar_data = json.load( fp ) _build_roar_scenario_index( _roar_scenarios, roar_data, None ) # try to connect each ASA scenario to ROAR diff --git a/vasl_templates/tools/make_chapter_h_placeholders.py b/vasl_templates/tools/make_chapter_h_placeholders.py index b08d2b7..62e8c47 100755 --- a/vasl_templates/tools/make_chapter_h_placeholders.py +++ b/vasl_templates/tools/make_chapter_h_placeholders.py @@ -38,9 +38,10 @@ def make_chapter_h_placeholders( output_fname, log=None \ results = {} # load the nationalities - fname = os.path.join( os.path.split(__file__)[0], "../webapp/data/default-template-pack/nationalities.json" ) global nationalities - nationalities = json.load( open( fname, "r" ) ) + fname = os.path.join( os.path.split(__file__)[0], "../webapp/data/default-template-pack/nationalities.json" ) + with open( fname, "r", encoding="utf-8" ) as fp: + nationalities = json.load( fp ) # load the vehicle/ordnance data files base_dir = os.path.join( os.path.split(__file__)[0], "../webapp/data/" ) @@ -95,16 +96,16 @@ def make_chapter_h_placeholders( output_fname, log=None \ base_dir = os.path.join( os.path.split(__file__)[0], "../webapp/data/extensions" ) for fname in glob.glob( os.path.join( base_dir, "*.json" ) ): extn_data = load_vo_data_from_extension( fname ) - for nat in extn_data: - for vo_type in extn_data[nat]: - for key in extn_data[nat][vo_type]: + for nat, vo_types in extn_data.items(): + for vo_type in vo_types: + for key in vo_types[vo_type]: if nat not in results: results[nat] = {} if vo_type not in results[nat]: results[nat][vo_type] = {} if key not in results[nat][vo_type]: results[nat][vo_type][key] = [] - results[nat][vo_type][key].extend( extn_data[nat][vo_type].get( key, [] ) ) + results[nat][vo_type][key].extend( vo_types[vo_type].get( key, [] ) ) # FUDGE! Allied Ordnance Note D is not in the Allied Minor common.json file (it's referenced # by some of the nationality-specific Guns e.g. Belgian DBT), so we add it in manually. @@ -154,7 +155,8 @@ def load_vo_data( fname, nat ): notes, ma_notes = set(), set() # load the file - vo_data = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + vo_data = json.load( fp ) for vo_entry in vo_data: if "note_number" in vo_entry: notes.add( @@ -218,7 +220,8 @@ def load_vo_data_from_extension( fname ): results = {} # get the extension ID - data = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + data = json.load( fp ) extn_id = data["extensionId"] if extn_id == "08d": # NOTE: All the vehicle/ordnance notes and multi-applicable notes in the Fight For Seoul extension diff --git a/vasl_templates/tools/tests/test_make_chapter_h_placeholders.py b/vasl_templates/tools/tests/test_make_chapter_h_placeholders.py index 8f1ff31..9317787 100644 --- a/vasl_templates/tools/tests/test_make_chapter_h_placeholders.py +++ b/vasl_templates/tools/tests/test_make_chapter_h_placeholders.py @@ -18,7 +18,8 @@ def test_make_chapter_h_placeholders(): # get the expected results fname = os.path.join( os.path.split(__file__)[0], "fixtures/chapter-h-placeholders.txt" ) - expected = [ line.strip() for line in open(fname,"r") ] + with open( fname, "r", encoding="utf-8" ) as fp: + expected = [ line.strip() for line in fp ] # check the results with ZipFile( temp_file.name, "r" ) as zip_file: diff --git a/vasl_templates/tools/webdriver_stress_test.py b/vasl_templates/tools/webdriver_stress_test.py index a0e3267..b99ae5c 100755 --- a/vasl_templates/tools/webdriver_stress_test.py +++ b/vasl_templates/tools/webdriver_stress_test.py @@ -44,7 +44,8 @@ def main( webapp_url, snippet_images, update_vsav, vsav_fname ): # read the VASL scenario file vsav_data = None if update_vsav > 0: - vsav_data = open( vsav_fname, "rb" ).read() + with open( vsav_fname, "rb" ) as fp: + vsav_data = fp.read() # prepare the test threads threads = [] @@ -185,7 +186,8 @@ def update_vsav_thread( webapp_url, vsav_fname, vsav_data ): # load a test scenario fname = os.path.join( os.path.split(__file__)[0], "../webapp/tests/fixtures/update-vsav/full.json" ) - saved_scenario = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + saved_scenario = json.load( fp ) load_scenario( saved_scenario, webdriver ) while not shutdown_event.is_set(): diff --git a/vasl_templates/utils.py b/vasl_templates/utils.py index eb2bfd8..04081ee 100644 --- a/vasl_templates/utils.py +++ b/vasl_templates/utils.py @@ -17,7 +17,8 @@ def get_build_info(): fname = os.path.join( BASE_DIR, "config", "build-info.json" ) if not os.path.isfile( fname ): return None - build_info = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + build_info = json.load( fp ) # get the build timestamp result = { "timestamp": build_info["timestamp"] } diff --git a/vasl_templates/webapp/__init__.py b/vasl_templates/webapp/__init__.py index 27bef31..97a43e5 100644 --- a/vasl_templates/webapp/__init__.py +++ b/vasl_templates/webapp/__init__.py @@ -176,7 +176,8 @@ def _on_sigint( signum, stack ): #pylint: disable=unused-argument # NOTE: os.path.isfile() and .exists() both return True even after the log file has gone!?!? # Is somebody caching something somewhere? :-/ try: - open( _LOCK_FNAME, "r" ) + with open( _LOCK_FNAME, "rb" ): + pass except FileNotFoundError: break time.sleep( 0.1 ) @@ -191,7 +192,8 @@ flask.cli.show_server_banner = lambda *args: None app = Flask( __name__ ) if _is_flask_child_process(): # we are the Flask child process - create a lock file - open( _LOCK_FNAME, "w" ).close() + with open( _LOCK_FNAME, "wb" ): + pass # set config defaults # NOTE: These are defined here since they are used by both the back- and front-ends. @@ -237,8 +239,8 @@ if os.path.isfile( _fname ): with open( _fname, "r", encoding="utf-8" ) as fp: try: logging.config.dictConfig( yaml.safe_load( fp ) ) - except Exception as ex: #pylint: disable=broad-except - logging.error( "Can't load the logging config: %s", ex ) + except Exception as _ex: #pylint: disable=broad-except + logging.error( "Can't load the logging config: %s", _ex ) else: # stop Flask from logging every request :-/ logging.getLogger( "werkzeug" ).setLevel( logging.WARNING ) diff --git a/vasl_templates/webapp/downloads.py b/vasl_templates/webapp/downloads.py index af3554f..b3666bd 100644 --- a/vasl_templates/webapp/downloads.py +++ b/vasl_templates/webapp/downloads.py @@ -141,12 +141,12 @@ class DownloadedFile: _logger.debug( "- If-None-Match = %s", _etags[url] ) headers[ "If-None-Match" ] = _etags[ url ] req = urllib.request.Request( url, headers=headers ) - resp = urllib.request.urlopen( req ) - data = resp.read() - if resp.headers.get( "Content-Encoding" ) == "gzip": - data = gzip.decompress( data ) - data = data.decode( "utf-8" ) - etag = resp.headers.get( "ETag" ) + with urllib.request.urlopen( req ) as resp: + resp_data = resp.read() + if resp.headers.get( "Content-Encoding" ) == "gzip": + resp_data = gzip.decompress( resp_data ) + data = resp_data.decode( "utf-8" ) + etag = resp.headers.get( "ETag" ) _logger.info( "Downloaded the %s file OK: %d bytes", df.key, len(data) ) if etag: _logger.debug( "- Got etag: %s", etag ) diff --git a/vasl_templates/webapp/files.py b/vasl_templates/webapp/files.py index f569f6a..1c82dad 100644 --- a/vasl_templates/webapp/files.py +++ b/vasl_templates/webapp/files.py @@ -32,15 +32,15 @@ class FileServer: url = "{}/{}".format( self.base_dir, path ) # NOTE: We download the target file and serve it ourself (instead of just redirecting) # since VASSAL can't handle SSL :-/ - resp = urllib.request.urlopen( url ) buf = io.BytesIO() - buf.write( resp.read() ) - buf.seek( 0 ) + with urllib.request.urlopen( url ) as resp: + buf.write( resp.read() ) mime_type = mimetypes.guess_type( url )[0] if not mime_type: # FUDGE! send_file() requires a MIME type, so we take a guess and hope the browser # can figure it out if we're wrong :-/ mime_type = "image/png" + buf.seek( 0 ) return send_file( buf, mimetype=mime_type ) else: path = path.replace( "\\", "/" ) # nb: for Windows :-/ @@ -95,7 +95,7 @@ def get_counter_image( gpid, side, index=0 ): abort( 404 ) return send_file( io.BytesIO( image_data ), - attachment_filename = os.path.split( image_path )[1] # nb: so Flask can figure out the MIME type + download_name = os.path.split( image_path )[1] # nb: so Flask can figure out the MIME type ) # --------------------------------------------------------------------- diff --git a/vasl_templates/webapp/main.py b/vasl_templates/webapp/main.py index b8aacbc..d80c67a 100644 --- a/vasl_templates/webapp/main.py +++ b/vasl_templates/webapp/main.py @@ -150,7 +150,7 @@ def get_app_config(): with open( fname, "r", encoding="utf-8" ) as fp: try: vals[ "SCENARIOS_CONFIG" ] = json.load( fp ) - except json.decoder.JSONDecodeError as ex: + except json.decoder.JSONDecodeError: msg = "Couldn't load the ASL Scenario Archive config." logging.error( "%s", msg ) startup_msg_store.error( msg ) @@ -219,7 +219,7 @@ def get_program_info(): "%H:%M %d %b %Y" ) params[ "DOCKER_CONTAINER_NAME" ] = os.environ.get( "DOCKER_CONTAINER_NAME" ) - with open( "/proc/self/cgroup", "r" ) as fp: + with open( "/proc/self/cgroup", "r", encoding="utf-8" ) as fp: buf = fp.read() mo = re.search( r"^\d+:name=.+/docker/([0-9a-f]{12})", buf, re.MULTILINE ) # NOTE: Reading cgroup stopped working when we upgraded to Fedora 33, but still works diff --git a/vasl_templates/webapp/run_server.py b/vasl_templates/webapp/run_server.py index 36b4f73..7fd2c69 100755 --- a/vasl_templates/webapp/run_server.py +++ b/vasl_templates/webapp/run_server.py @@ -62,7 +62,8 @@ def main( bind_addr, force_init_delay, flask_debug ): # (in particular, starting the download thread). time.sleep( force_init_delay ) url = "http://{}:{}/ping".format( host, port ) - _ = urllib.request.urlopen( url ) + with urllib.request.urlopen( url ) as resp: + _ = resp.read() threading.Thread( target=_start_server, daemon=True ).start() # run the server diff --git a/vasl_templates/webapp/scenarios.py b/vasl_templates/webapp/scenarios.py index 4706e52..1e601a2 100644 --- a/vasl_templates/webapp/scenarios.py +++ b/vasl_templates/webapp/scenarios.py @@ -188,8 +188,8 @@ def get_scenario( scenario_id ): #pylint: disable=too-many-locals # get any files available for download downloads = {} keys = { "vt_setup": "vaslTemplates", "vasl_setup": "vaslTemplateSetups", "screenshot": "templateImages" } - for key in keys: - for entry in scenario.get( keys[key], [] ): + for key, ftype in keys.items(): + for entry in scenario.get( ftype, [] ): fname = os.path.basename( entry.get( "url", "" ) ) pos = fname.find( "|" ) if pos < 0: @@ -563,8 +563,8 @@ def on_successful_asa_upload( scenario_id ): # download the specified scenario url = app.config["ASA_GET_SCENARIO_URL"].replace( "{ID}", scenario_id ) try: - fp = urllib.request.urlopen( url ) - new_scenario = json.loads( fp.read().decode( "utf-8" ) ) + with urllib.request.urlopen( url ) as fp: + new_scenario = json.loads( fp.read().decode( "utf-8" ) ) except Exception as ex: #pylint: disable=broad-except msg = str( getattr(ex,"reason",None) or ex ) return jsonify( { "status": "error", "message": msg } ) @@ -618,8 +618,8 @@ def test_asa_upload( scenario_id ): """Generate a response.""" dname = os.path.join( os.path.dirname(__file__), "tests/fixtures/asa-responses/" ) fname = os.path.join( dname, "{}.json".format( fname ) ) - resp = json.load( open( fname, "r", encoding="utf-8" ) ) - return jsonify( resp ) + with open( fname, "r", encoding="utf-8" ) as fp: + return jsonify( json.load( fp ) ) # simulate a slow response delay = parse_int( app.config.get( "ASA_UPLOAD_DELAY" ), 0 ) diff --git a/vasl_templates/webapp/snippets.py b/vasl_templates/webapp/snippets.py index cdef7ff..858f383 100644 --- a/vasl_templates/webapp/snippets.py +++ b/vasl_templates/webapp/snippets.py @@ -204,10 +204,11 @@ def get_flag( nat ): fname = globvars.template_pack.get( "nationalities", {} ).get( nat, {} ).get( "flag" ) if fname: if fname.startswith( ("http://","https://") ): - fp = urllib.request.urlopen( fname ) + with urllib.request.urlopen( fname ) as fp: + return _get_small_image( fp, key, height ) else: - fp = open( fname, "rb" ) - return _get_small_image( fp, key, height ) + with open( fname, "rb" ) as fp: + return _get_small_image( fp, key, height ) # serve the standard flag fname = os.path.join( "static/images/flags/", nat+".png" ) diff --git a/vasl_templates/webapp/static/help/index.html b/vasl_templates/webapp/static/help/index.html index 6d9ee52..65225be 100644 --- a/vasl_templates/webapp/static/help/index.html +++ b/vasl_templates/webapp/static/help/index.html @@ -65,7 +65,7 @@ pip install .[gui]

If you're on Windows, the Qt runtime will have been installed as part of PyQt5 (when you did the pip install above), but if you're in a virtual environment and you're getting "DLL load failed" errors, this is due to a problem with the way Python sets up the virtualenv. In the virtualenv's scripts/ sub-directory, there should be two Python DLL's, so if you're missing python3.dll, copy it over from the Python installation the virtualenv was created from, and you should be good to go. -

If you're on Linux, you may need to install Qt 5.15.2. On Fedora 33, running the "gui" install above should install everything you need. +

If you're on Linux, you may need to install Qt 5.15.4. On Fedora 33, running the "gui" install above should install everything you need.

Then, just run the vasl-templates command. diff --git a/vasl_templates/webapp/tests/control_tests_servicer.py b/vasl_templates/webapp/tests/control_tests_servicer.py index fd27495..915ae26 100644 --- a/vasl_templates/webapp/tests/control_tests_servicer.py +++ b/vasl_templates/webapp/tests/control_tests_servicer.py @@ -128,7 +128,7 @@ class ControlTestsServicer( BaseControlTestsServicer ): #pylint: disable=too-man # set up a directory for our temp files if self._temp_dir: self._temp_dir.cleanup() - self._temp_dir = tempfile.TemporaryDirectory() + self._temp_dir = tempfile.TemporaryDirectory() #pylint: disable=consider-using-with # reset the webapp server ctx = None diff --git a/vasl_templates/webapp/tests/test_counters.py b/vasl_templates/webapp/tests/test_counters.py index d92ab63..aeaa638 100644 --- a/vasl_templates/webapp/tests/test_counters.py +++ b/vasl_templates/webapp/tests/test_counters.py @@ -30,9 +30,9 @@ def test_counter_images( webapp, webdriver ): #pylint: disable=too-many-locals for side in ("front","back"): url = webapp.url_for( "get_counter_image", gpid=gpid, side=side ) try: - resp = urllib.request.urlopen( url ) - resp_code = resp.code - resp_data = resp.read() + with urllib.request.urlopen( url ) as resp: + resp_code = resp.code + resp_data = resp.read() except urllib.error.HTTPError as ex: resp_code = ex.code resp_data = None @@ -45,7 +45,8 @@ def test_counter_images( webapp, webdriver ): #pylint: disable=too-many-locals # a missing image for everything anyway. We just use the most recent supported version. gpids = get_vo_gpids( None ) fname = os.path.join( os.path.split(__file__)[0], "../static/images/missing-image.png" ) - missing_image_data = open( fname, "rb" ).read() + with open( fname, "rb" ) as fp: + missing_image_data = fp.read() check_images( gpids, check_front = lambda code, data: code == 200 and data == missing_image_data, check_back = lambda code, data: code == 200 and data == missing_image_data @@ -77,13 +78,14 @@ def test_counter_images( webapp, webdriver ): #pylint: disable=too-many-locals # figure out what we're expecting to see fname = os.path.join( check_dir, "vasl-pieces-{}.txt".format( vasl_version ) ) - expected_vasl_pieces = open( fname, "r" ).read() + with open( fname, "r", encoding="utf-8" ) as fp: + expected_vasl_pieces = fp.read() # generate a report for the pieces loaded report, gpids = webapp.control_tests.get_vasl_pieces( vasl_version ) if save_dir: fname2 = os.path.join( save_dir, vasl_version+".txt" ) - with open( fname2, "w" ) as fp: + with open( fname2, "w", encoding="utf-8" ) as fp: fp.write( report ) # check the report @@ -114,8 +116,8 @@ def _DISABLED_test_gpid_remapping( webapp, webdriver ): """Check if we can get the image for the specified GPID.""" url = webapp.url_for( "get_counter_image", gpid=gpid, side="front" ) try: - resp = urllib.request.urlopen( url ) - return resp.code + with urllib.request.urlopen( url ) as resp: + return resp.code except urllib.error.HTTPError as ex: assert ex.code != 200 return ex.code @@ -156,7 +158,8 @@ def _DISABLED_test_gpid_remapping( webapp, webdriver ): # load the test scenario fname = os.path.join( os.path.split(__file__)[0], "fixtures/gpid-remapping.json" ) - scenario_data = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + scenario_data = json.load( fp ) # run the tests using VASL 6.4.4 and 6.5.0 # NOTE: Versions of VASL prior to 6.6.0 are no longer officially supported (since they use Java 8), diff --git a/vasl_templates/webapp/tests/test_files.py b/vasl_templates/webapp/tests/test_files.py index 43e4950..099af2e 100644 --- a/vasl_templates/webapp/tests/test_files.py +++ b/vasl_templates/webapp/tests/test_files.py @@ -87,11 +87,11 @@ def test_local_user_files( webapp, webdriver ): # try getting a user file try: url = webapp.url_for( "get_user_file", path="hello.txt" ) - resp = urllib.request.urlopen( url ) - assert enable_user_files # nb: we should only get here if user files are enabled - assert resp.code == 200 - assert resp.read().strip() == b"Yo, wassup!" - assert resp.headers[ "Content-Type" ].startswith( "text/plain" ) + with urllib.request.urlopen( url ) as resp: + assert enable_user_files # nb: we should only get here if user files are enabled + assert resp.code == 200 + assert resp.read().strip() == b"Yo, wassup!" + assert resp.headers[ "Content-Type" ].startswith( "text/plain" ) except urllib.error.HTTPError as ex: assert not enable_user_files # nb: we should only get here if user files are disabled assert ex.code == 404 @@ -99,17 +99,18 @@ def test_local_user_files( webapp, webdriver ): # try getting a non-existent file (nb: should always fail, whether user files are enabled/disabled) with pytest.raises( urllib.error.HTTPError ) as exc_info: url = webapp.url_for( "get_user_file", path="unknown" ) - resp = urllib.request.urlopen( url ) + with urllib.request.urlopen( url ): + pass assert exc_info.value.code == 404 # try getting a file in a sub-directory try: url = webapp.url_for( "get_user_file", path="subdir/placeholder.png" ) - resp = urllib.request.urlopen( url ) - assert enable_user_files # nb: we should only get here if user files are enabled - assert resp.code == 200 - assert resp.read().startswith( b"\x89PNG\r\n" ) - assert resp.headers[ "Content-Type" ] == "image/png" + with urllib.request.urlopen( url ) as resp: + assert enable_user_files # nb: we should only get here if user files are enabled + assert resp.code == 200 + assert resp.read().startswith( b"\x89PNG\r\n" ) + assert resp.headers[ "Content-Type" ] == "image/png" except urllib.error.HTTPError as ex: assert not enable_user_files # nb: we should only get here if user files are disabled assert ex.code == 404 @@ -119,17 +120,18 @@ def test_local_user_files( webapp, webdriver ): assert os.path.isfile( fname ) with pytest.raises( urllib.error.HTTPError ) as exc_info: url = webapp.url_for( "get_user_file", path="../new-default-scenario.json" ) - resp = urllib.request.urlopen( url ) + with urllib.request.urlopen( url ): + pass assert exc_info.value.code == 404 # try getting a file with special characters in its name try: url = webapp.url_for( "get_user_file", path="amp=& ; plus=+.txt" ) - resp = urllib.request.urlopen( url ) - assert enable_user_files # nb: we should only get here if user files are enabled - assert resp.code == 200 - assert resp.read().strip() == b"special chars" - assert resp.headers[ "Content-Type" ].startswith( "text/plain" ) + with urllib.request.urlopen( url ) as resp: + assert enable_user_files # nb: we should only get here if user files are enabled + assert resp.code == 200 + assert resp.read().strip() == b"special chars" + assert resp.headers[ "Content-Type" ].startswith( "text/plain" ) except urllib.error.HTTPError as ex: assert not enable_user_files # nb: we should only get here if user files are disabled assert ex.code == 404 @@ -164,11 +166,11 @@ def test_remote_user_files( webapp, webdriver ): # try getting a user file try: url = webapp.url_for( "get_user_file", path="menu.png" ) - resp = urllib.request.urlopen( url ) - assert enable_user_files # nb: we should only get here if user files are enabled - assert resp.code == 200 - assert resp.read().startswith( b"\x89PNG\r\n" ) - assert resp.headers[ "Content-Type" ] == "image/png" + with urllib.request.urlopen( url ) as resp: + assert enable_user_files # nb: we should only get here if user files are enabled + assert resp.code == 200 + assert resp.read().startswith( b"\x89PNG\r\n" ) + assert resp.headers[ "Content-Type" ] == "image/png" except urllib.error.HTTPError as ex: assert not enable_user_files # nb: we should only get here if user files are disabled assert ex.code == 404 diff --git a/vasl_templates/webapp/tests/test_lfa.py b/vasl_templates/webapp/tests/test_lfa.py index 75634ee..43af3f5 100644 --- a/vasl_templates/webapp/tests/test_lfa.py +++ b/vasl_templates/webapp/tests/test_lfa.py @@ -617,7 +617,8 @@ def _analyze_vlogs( fnames ): # add each log file for fno,fname in enumerate(fnames): fname = os.path.join( os.path.split(__file__)[0], "fixtures/analyze-vlog/"+fname ) - vlog_data = open( fname, "rb" ).read() + with open( fname, "rb" ) as fp: + vlog_data = fp.read() set_stored_msg( "_vlog-persistence_", "{}|{}".format( os.path.split( fname )[1], base64.b64encode( vlog_data ).decode( "utf-8" ) diff --git a/vasl_templates/webapp/tests/test_national_capabilities.py b/vasl_templates/webapp/tests/test_national_capabilities.py index e175298..e14b9a6 100644 --- a/vasl_templates/webapp/tests/test_national_capabilities.py +++ b/vasl_templates/webapp/tests/test_national_capabilities.py @@ -49,17 +49,18 @@ def test_national_capabilities_reports( webapp, webdriver ): if save_dir: fname2 = os.path.join( save_dir, fname ) os.makedirs( os.path.split(fname2)[0], exist_ok=True ) - with open( os.path.join(save_dir,fname2), "w" ) as fp: + with open( os.path.join(save_dir,fname2), "w", encoding="utf-8" ) as fp: fp.write( report ) # check the report fname = os.path.join( check_dir, fname ) - if open(fname,"r",encoding="utf-8").read() != report: - if save_dir: - print( "FAILED:", fname ) - failed = True - else: - assert False, "Report mismatch: {}".format( fname ) + with open( fname, "r", encoding="utf-8" ) as fp: + if fp.read() != report: + if save_dir: + print( "FAILED:", fname ) + failed = True + else: + assert False, "Report mismatch: {}".format( fname ) assert not failed diff --git a/vasl_templates/webapp/tests/test_scenario_persistence.py b/vasl_templates/webapp/tests/test_scenario_persistence.py index 8b3e6b0..2e1b832 100644 --- a/vasl_templates/webapp/tests/test_scenario_persistence.py +++ b/vasl_templates/webapp/tests/test_scenario_persistence.py @@ -186,9 +186,9 @@ def test_scenario_persistence( webapp, webdriver ): #pylint: disable=too-many-st load_scenario( saved_scenario ) check_window_title( "my test scenario (xyz123)" ) check_ob_tabs( "russian", "german" ) - for tab_id in SCENARIO_PARAMS: + for tab_id, params in SCENARIO_PARAMS.items(): select_tab( tab_id ) - for field,val in SCENARIO_PARAMS[tab_id].items(): + for field, val in params.items(): if field in ("SCENARIO_NOTES","SSR"): continue # nb: these require special handling, we do it below if field in ("OB_SETUPS_1","OB_SETUPS_2","OB_NOTES_1","OB_NOTES_2"): diff --git a/vasl_templates/webapp/tests/test_scenario_search.py b/vasl_templates/webapp/tests/test_scenario_search.py index 19abbc5..1a4e527 100644 --- a/vasl_templates/webapp/tests/test_scenario_search.py +++ b/vasl_templates/webapp/tests/test_scenario_search.py @@ -612,7 +612,8 @@ def test_scenario_upload( webapp, webdriver ): assert asa_upload["token"] == api_token # send the VSAV data to the front-end fname = os.path.join( os.path.dirname(__file__), "fixtures/update-vsav/full.vsav" ) - vsav_data = open( fname, "rb" ).read() + with open( fname, "rb" ) as fp: + vsav_data = fp.read() set_stored_msg( "_vsav-persistence_", base64.b64encode( vsav_data ).decode( "utf-8" ) ) find_child( ".vsav-container", dlg ).click() # wait for the files to be prepared diff --git a/vasl_templates/webapp/tests/test_template_packs.py b/vasl_templates/webapp/tests/test_template_packs.py index bdaa9e0..205fa1d 100644 --- a/vasl_templates/webapp/tests/test_template_packs.py +++ b/vasl_templates/webapp/tests/test_template_packs.py @@ -186,6 +186,32 @@ def test_missing_templates( webapp, webdriver ): else: return template_id + def check_buttons( fname, sel, is_snippet_control ): #pylint: disable=missing-docstring + for btn in find_children( sel ): + # check the UI state of the next button + template_id = adjust_template_id( btn.get_attribute( "data-id" ) ) + if fname == "national-capabilities.json": + expected = False # nb: this is the JSON file, not the template file, and so doesn't effect buttons + elif fname == "nat_caps.j2": + expected = template_id.startswith( "nat_caps_" ) + else: + expected = os.path.splitext( fname )[0] == template_id + disabled = webdriver.execute_script( "return $(arguments[0]).button('option','disabled')", btn ) + assert expected == disabled + # check that snippet control groups have been enabled/disabled correctly + parent = btn.find_element_by_xpath( ".." ) + parent_classes = get_css_classes( parent ) + if is_snippet_control: + assert "snippet-control" in parent_classes + elem = find_child( ".ui-selectmenu-button", parent ) + elem_classes = get_css_classes( elem ) + if expected: + assert "ui-selectmenu-disabled" in elem_classes + else: + assert "ui-selectmenu-disabled" not in elem_classes + else: + assert "snippet-control" not in parent_classes + # upload the template pack, with one file missing each time for fname in files: @@ -197,33 +223,8 @@ def test_missing_templates( webapp, webdriver ): # check the state of each button (everything should be enabled, except for the one # corresponding to the template file we excluded from the upload) - def check_buttons( sel, is_snippet_control ): #pylint: disable=missing-docstring - for btn in find_children( sel ): - # check the UI state of the next button - template_id = adjust_template_id( btn.get_attribute( "data-id" ) ) - if fname == "national-capabilities.json": - expected = False # nb: this is the JSON file, not the template file, and so doesn't effect buttons - elif fname == "nat_caps.j2": - expected = template_id.startswith( "nat_caps_" ) - else: - expected = os.path.splitext( fname )[0] == template_id - disabled = webdriver.execute_script( "return $(arguments[0]).button('option','disabled')", btn ) - assert expected == disabled - # check that snippet control groups have been enabled/disabled correctly - parent = btn.find_element_by_xpath( ".." ) - parent_classes = get_css_classes( parent ) - if is_snippet_control: - assert "snippet-control" in parent_classes - elem = find_child( ".ui-selectmenu-button", parent ) - elem_classes = get_css_classes( elem ) - if expected: - assert "ui-selectmenu-disabled" in elem_classes - else: - assert "ui-selectmenu-disabled" not in elem_classes - else: - assert "snippet-control" not in parent_classes - check_buttons( "button.generate", True ) - check_buttons( "button.edit-template", False ) + check_buttons( fname, "button.generate", True ) + check_buttons( fname, "button.edit-template", False ) # NOTE: We should really check that the "generate snippet" buttons don't appear in sortable entries, # but that's more trouble than it's worth - templates such as ob_setup and ob_vehicles are never @@ -250,7 +251,7 @@ def make_zip_from_files( dname ): fname = os.path.join( root, fname ) assert fname.startswith( dname ) fname2 = fname[len(dname)+1:] - with open( fname, "r" ) as fp: + with open( fname, "r", encoding="utf-8" ) as fp: files[fname2] = fp.read() return _make_zip( files ) diff --git a/vasl_templates/webapp/tests/test_vasl_extensions.py b/vasl_templates/webapp/tests/test_vasl_extensions.py index 86ae104..97bbca1 100644 --- a/vasl_templates/webapp/tests/test_vasl_extensions.py +++ b/vasl_templates/webapp/tests/test_vasl_extensions.py @@ -63,7 +63,8 @@ def test_vasl_extension_info( webapp, webdriver ): # prepare our test VASL extension fname = os.path.join( os.path.split(__file__)[0], "fixtures/vasl-extensions/test-extn.xml" ) - extn_fname = _set_test_vasl_extn( webapp, open(fname,"r").read() ) + with open( fname, "r", encoding="utf=8" ) as fp: + extn_fname = _set_test_vasl_extn( webapp, fp.read() ) def do_test( dname, expected ): #pylint: disable=missing-docstring webapp.control_tests \ diff --git a/vasl_templates/webapp/tests/test_vassal.py b/vasl_templates/webapp/tests/test_vassal.py index b12b198..9b217b2 100644 --- a/vasl_templates/webapp/tests/test_vassal.py +++ b/vasl_templates/webapp/tests/test_vassal.py @@ -341,7 +341,8 @@ def test_dump_vsav( webapp, webdriver ): # check the result fname = change_extn( fname, ".txt" ) - expected = open( fname, "r" ).read() + with open( fname, "r", encoding="utf-8" ) as fp: + expected = fp.read() assert vsav_dump == expected # run the test against all versions of VASSAL+VASL @@ -370,7 +371,8 @@ def test_update_legacy_labels( webapp, webdriver ): # load the scenario into the UI and update the VSAV fname2 = change_extn( fname, ".json" ) - saved_scenario = json.load( open( fname2, "r" ) ) + with open( fname2, "r", encoding="utf-8" ) as fp: + saved_scenario = json.load( fp ) load_scenario( saved_scenario ) expected = 5 if enable_vo_notes else 1 updated_vsav_dump = _update_vsav_and_dump( webapp, fname, diff --git a/vasl_templates/webapp/tests/test_vehicles_ordnance.py b/vasl_templates/webapp/tests/test_vehicles_ordnance.py index 6c4cc94..bf11948 100644 --- a/vasl_templates/webapp/tests/test_vehicles_ordnance.py +++ b/vasl_templates/webapp/tests/test_vehicles_ordnance.py @@ -359,7 +359,8 @@ def test_common_vo( webapp, webdriver ): #pylint: disable=too-many-locals def get_common_vo( fname ): """Get the vehicle/ordnance information from the specified file.""" fname = os.path.join( DATA_DIR, fname ) - data = json.load( open( fname, "r" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + data = json.load( fp ) def get_gpid( val ): #pylint: disable=missing-docstring if isinstance( val, list ): val = val[0] @@ -750,7 +751,8 @@ def test_invalid_vo_image_ids( webapp, webdriver ): continue # load the next scenario, make sure a warning was issued for the V/O image ID - data = json.load( open(fname,"r") ) + with open( fname, "r", encoding="utf-8" ) as fp: + data = json.load( fp ) set_stored_msg_marker( "_last-warning_" ) load_scenario( data ) last_warning = get_stored_msg( "_last-warning_" ) diff --git a/vasl_templates/webapp/tests/test_vo_notes.py b/vasl_templates/webapp/tests/test_vo_notes.py index fdb5d9b..1541e08 100644 --- a/vasl_templates/webapp/tests/test_vo_notes.py +++ b/vasl_templates/webapp/tests/test_vo_notes.py @@ -366,28 +366,28 @@ def test_vo_notes_image_cache( webapp, webdriver ): url = mo.group( 1 ) # get the vehicle note image (should be created) - resp = urllib.request.urlopen( url ) - assert not resp.headers.get( "X-WasCached" ) - image_data = resp.read() + with urllib.request.urlopen( url ) as resp: + assert not resp.headers.get( "X-WasCached" ) + image_data = resp.read() # get the vehicle note image (should be re-created) - resp = urllib.request.urlopen( url ) - assert not resp.headers.get( "X-WasCached" ) - assert resp.read() == image_data + with urllib.request.urlopen( url ) as resp: + assert not resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data # enable image caching webapp.control_tests.set_app_config_val( "VO_NOTES_IMAGE_CACHE_DIR", "{{TEMP_DIR}}" ) init_test() # get the vehicle note image (should be re-created) - resp = urllib.request.urlopen( url ) - assert not resp.headers.get( "X-WasCached" ) - assert resp.read() == image_data + with urllib.request.urlopen( url ) as resp: + assert not resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data # get the vehicle note image (should be cached) - resp = urllib.request.urlopen( url ) - assert resp.headers.get( "X-WasCached" ) - assert resp.read() == image_data + with urllib.request.urlopen( url ) as resp: + assert resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data # --------------------------------------------------------------------- @@ -587,12 +587,13 @@ def test_vo_notes_reports( webapp, webdriver ): #pylint: disable=too-many-locals # check the report fname = os.path.join( check_dir, fname ) - if open( fname, "r", encoding="utf-8" ).read() != report: - if save_dir: - print( "FAILED:", fname ) - failed = True - else: - assert False, "Report mismatch: {}".format( fname ) + with open( fname, "r", encoding="utf-8" ) as fp: + if fp.read() != report: + if save_dir: + print( "FAILED:", fname ) + failed = True + else: + assert False, "Report mismatch: {}".format( fname ) assert not failed diff --git a/vasl_templates/webapp/tests/test_vo_reports.py b/vasl_templates/webapp/tests/test_vo_reports.py index 17b4eb6..70aa317 100644 --- a/vasl_templates/webapp/tests/test_vo_reports.py +++ b/vasl_templates/webapp/tests/test_vo_reports.py @@ -98,17 +98,18 @@ def test_vo_reports( webapp, webdriver ): #pylint: disable=too-many-locals if save_dir: fname2 = os.path.join( save_dir, fname ) os.makedirs( os.path.split(fname2)[0], exist_ok=True ) - with open( os.path.join(save_dir,fname2), "w" ) as fp: + with open( os.path.join(save_dir,fname2), "w", encoding="utf-8" ) as fp: fp.write( report ) # check the report fname = os.path.join( check_dir, fname ) - if open(fname,"r",encoding="utf-8").read() != report: - if save_dir: - print( "FAILED:", fname ) - failed = True - else: - assert False, "Report mismatch: {}".format( fname ) + with open( fname, "r", encoding="utf-8" ) as fp: + if fp.read() != report: + if save_dir: + print( "FAILED:", fname ) + failed = True + else: + assert False, "Report mismatch: {}".format( fname ) assert not failed diff --git a/vasl_templates/webapp/tests/utils.py b/vasl_templates/webapp/tests/utils.py index ec112ce..b7ffedf 100644 --- a/vasl_templates/webapp/tests/utils.py +++ b/vasl_templates/webapp/tests/utils.py @@ -327,7 +327,8 @@ def get_nationality_display_name( nat_id ): def get_nationalities( webapp ): """Get the nationalities table.""" url = webapp.url_for( "get_template_pack" ) - template_pack = json.load( urllib.request.urlopen( url ) ) + with urllib.request.urlopen( url ) as resp: + template_pack = json.load( resp ) return template_pack["nationalities"] # --------------------------------------------------------------------- diff --git a/vasl_templates/webapp/utils.py b/vasl_templates/webapp/utils.py index 688c994..cbd2bac 100644 --- a/vasl_templates/webapp/utils.py +++ b/vasl_templates/webapp/utils.py @@ -75,7 +75,7 @@ class TempFile: else: encoding = "utf-8" if "b" not in self.mode else None assert self.temp_file is None - self.temp_file = tempfile.NamedTemporaryFile( + self.temp_file = tempfile.NamedTemporaryFile( #pylint: disable=consider-using-with mode = self.mode, encoding = encoding, suffix = self.extn, diff --git a/vasl_templates/webapp/vasl_mod.py b/vasl_templates/webapp/vasl_mod.py index b02ece6..1c1fe17 100644 --- a/vasl_templates/webapp/vasl_mod.py +++ b/vasl_templates/webapp/vasl_mod.py @@ -103,12 +103,11 @@ def _load_vasl_extns( extn_dir, msg_store ): #pylint: disable=too-many-locals,to # try to load the extension _logger.debug( "Checking VASL extension: %s", extn_fname ) try: - zip_file = zipfile.ZipFile( extn_fname, "r" ) + with zipfile.ZipFile( extn_fname, "r" ) as zf: + build_info = zf.read( "buildFile" ) except zipfile.BadZipFile: log_warning( "Can't check VASL extension (not a ZIP file): {}", extn_fname ) continue - try: - build_info = zip_file.read( "buildFile" ) except KeyError: log_warning( "Missing buildFile: {}", extn_fname ) continue @@ -163,11 +162,11 @@ class VaslMod: # initialize self._pieces = {} - self._files = [ ( zipfile.ZipFile(fname,"r"), None ) ] + self._files = [ ( zipfile.ZipFile(fname,"r"), None ) ] #pylint: disable=consider-using-with if extns: for extn in extns: self._files.append( - ( zipfile.ZipFile(extn[0],"r"), extn[1] ) + ( zipfile.ZipFile(extn[0],"r"), extn[1] ) #pylint: disable=consider-using-with ) # load the VASL module and any extensions @@ -258,9 +257,11 @@ class VaslMod: # load our overrides fname = os.path.join( data_dir, "vasl-"+self.vasl_version, "vasl-overrides.json" ) - vasl_overrides = json.load( open( fname, "r", encoding="utf-8" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + vasl_overrides = json.load( fp ) fname = os.path.join( data_dir, "vasl-"+self.vasl_version, "expected-multiple-images.json" ) - expected_multiple_images = json.load( open( fname, "r", encoding="utf-8" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + expected_multiple_images = json.load( fp ) # figure out which pieces we're interested in target_gpids = get_vo_gpids( self ) diff --git a/vasl_templates/webapp/vassal.py b/vasl_templates/webapp/vassal.py index 9145dc3..5a73a55 100644 --- a/vasl_templates/webapp/vassal.py +++ b/vasl_templates/webapp/vassal.py @@ -464,18 +464,20 @@ class VassalShim: # but it does hide the DOS box if the user has configured java.exe instead of javaw.exe. kwargs["creationflags"] = 0x8000000 # nb: win32process.CREATE_NO_WINDOW try: - proc = subprocess.Popen( args2, **kwargs ) + with subprocess.Popen( args2, **kwargs ) as proc: + try: + proc.wait( timeout ) + except subprocess.TimeoutExpired: + proc.kill() + raise except FileNotFoundError as ex: raise SimpleError( "Can't run the VASSAL shim (have you configured Java?): {}".format( ex ) ) from ex - try: - proc.wait( timeout ) - except subprocess.TimeoutExpired: - proc.kill() - raise buf1.close( delete=False ) - stdout = open( buf1.name, "r", encoding="utf-8" ).read() + with open( buf1.name, "r", encoding="utf-8" ) as fp: + stdout = fp.read() buf2.close( delete=False ) - stderr = open( buf2.name, "r", encoding="utf-8" ).read() + with open( buf2.name, "r", encoding="utf-8" ) as fp: + stderr = fp.read() elapsed_time = time.time() - start_time logger.info( "- Completed OK: %.3fs", elapsed_time ) diff --git a/vasl_templates/webapp/vo_notes.py b/vasl_templates/webapp/vo_notes.py index a10cb0a..46f5799 100644 --- a/vasl_templates/webapp/vo_notes.py +++ b/vasl_templates/webapp/vo_notes.py @@ -202,17 +202,17 @@ def load_vo_notes( msg_store ): #pylint: disable=too-many-statements,too-many-lo vo_notes[ vo_type2 ][ nat2 ][ "multi-applicable" ] = ma_notes # update nationality variants with the notes from their base nationality - for vo_type2 in vo_notes: + for vo_type2, vo_notes2 in vo_notes.items(): # FUDGE! Some nationalities don't have any vehicles/ordnance of their own, so we have to do this manually. # NOTE: We do a deep copy so that these new nationalities don't get affected by changes we make # to the base nationality later (e.g. adding K:FW counters to the British). - if "chinese" in vo_notes[vo_type2]: - vo_notes[vo_type2]["chinese~gmd"] = copy.deepcopy( vo_notes[vo_type2]["chinese"] ) - if "british" in vo_notes[vo_type2]: - vo_notes[vo_type2]["british~canadian"] = copy.deepcopy( vo_notes[vo_type2]["british"] ) - vo_notes[vo_type2]["british~newzealand"] = copy.deepcopy( vo_notes[vo_type2]["british"] ) - vo_notes[vo_type2]["british~australian"] = copy.deepcopy( vo_notes[vo_type2]["british"] ) - vo_notes[vo_type2]["british~anzac"] = copy.deepcopy( vo_notes[vo_type2]["british"] ) + if "chinese" in vo_notes2: + vo_notes2["chinese~gmd"] = copy.deepcopy( vo_notes2["chinese"] ) + if "british" in vo_notes2: + vo_notes2["british~canadian"] = copy.deepcopy( vo_notes2["british"] ) + vo_notes2["british~newzealand"] = copy.deepcopy( vo_notes2["british"] ) + vo_notes2["british~australian"] = copy.deepcopy( vo_notes2["british"] ) + vo_notes2["british~anzac"] = copy.deepcopy( vo_notes2["british"] ) def install_kfw_vo_notes( nat, vo_type, extn_id, include ): """Install the K:FW vehicle/ordnance notes into the specified nationality.""" @@ -426,10 +426,11 @@ def load_asl_rulebook2_vo_note_targets( msg_store ): return try: if os.path.isfile( base_url ): - fp = open( base_url, "r", encoding="utf-8" ) + with open( base_url, "r", encoding="utf-8" ) as fp: + _asl_rulebook2_targets = json.load( fp ) else: - fp = urllib.request.urlopen( base_url + "/vo-note-targets" ) - _asl_rulebook2_targets = json.load( fp ) + with urllib.request.urlopen( base_url + "/vo-note-targets" ) as fp: + _asl_rulebook2_targets = json.load( fp ) except Exception as ex: #pylint: disable=broad-except msg = str( getattr(ex,"reason",None) or ex ) msg_store.warning( "Couldn't get the ASL Rulebook2 Chapter H targets: {}".format( msg ) ) diff --git a/vasl_templates/webapp/vo_utils.py b/vasl_templates/webapp/vo_utils.py index 64ceaf9..280c151 100644 --- a/vasl_templates/webapp/vo_utils.py +++ b/vasl_templates/webapp/vo_utils.py @@ -206,7 +206,8 @@ def add_vo_comments( listings, vo_type, msg_store ): global _vo_comments if not _vo_comments: fname = os.path.join( app.config.get("DATA_DIR",DATA_DIR), "vo-comments.json" ) - _vo_comments = json.load( open( fname, "r", encoding="utf-8" ) ) + with open( fname, "r", encoding="utf-8" ) as fp: + _vo_comments = json.load( fp ) # process each vehicle/ordnance for nat,vo_entries in listings.items(): diff --git a/vasl_templates/webapp/webdriver.py b/vasl_templates/webapp/webdriver.py index 9d82e8a..c703fe1 100644 --- a/vasl_templates/webapp/webdriver.py +++ b/vasl_templates/webapp/webdriver.py @@ -39,7 +39,7 @@ class WebDriver: def start( self ): """Start the webdriver.""" - self.lock.acquire() + self.lock.acquire() #pylint: disable=consider-using-with self._do_start() def _do_start( self ):