From 089e43f31e5916d6c20749f12db4d956ecf23ac8 Mon Sep 17 00:00:00 2001 From: Taka Date: Wed, 23 Jun 2021 02:17:24 +1000 Subject: [PATCH] Added a cache for vehicle/ordnance note images. --- vasl_templates/webapp/__init__.py | 12 ++ vasl_templates/webapp/static/help/index.html | 1 + vasl_templates/webapp/static/snippets.js | 13 +- vasl_templates/webapp/static/vo.js | 48 ++++--- .../configure-vo-notes-image-cache.html | 7 + .../templates/load-vo-notes-image-cache.html | 136 ++++++++++++++++++ .../webapp/templates/vo-notes-report.html | 2 +- .../webapp/tests/control_tests_servicer.py | 3 + vasl_templates/webapp/tests/test_vo_notes.py | 80 +++++++++-- vasl_templates/webapp/vo_notes.py | 65 +++++++-- 10 files changed, 315 insertions(+), 52 deletions(-) create mode 100644 vasl_templates/webapp/templates/configure-vo-notes-image-cache.html create mode 100644 vasl_templates/webapp/templates/load-vo-notes-image-cache.html diff --git a/vasl_templates/webapp/__init__.py b/vasl_templates/webapp/__init__.py index eb3b961..27bef31 100644 --- a/vasl_templates/webapp/__init__.py +++ b/vasl_templates/webapp/__init__.py @@ -84,6 +84,18 @@ def _init_webapp(): from vasl_templates.webapp.vo_notes import load_vo_notes #pylint: disable=cyclic-import load_vo_notes( startup_msg_store ) + # initialize the vehicle/ordnance notes image cache + from vasl_templates.webapp import vo_notes as webapp_vo_notes #pylint: disable=reimported + dname = app.config.get( "VO_NOTES_IMAGE_CACHE_DIR" ) + if dname in ( "disable", "disabled" ): + webapp_vo_notes._vo_notes_image_cache_dname = None #pylint: disable=protected-access + elif dname: + webapp_vo_notes._vo_notes_image_cache_dname = dname #pylint: disable=protected-access + else: + webapp_vo_notes._vo_notes_image_cache_dname = os.path.join( #pylint: disable=protected-access + tempfile.gettempdir(), "vasl-templates", "vo-notes-image-cache" + ) + # load integration data from asl-rulebook2 from vasl_templates.webapp.vo_notes import load_asl_rulebook2_vo_note_targets #pylint: disable=cyclic-import load_asl_rulebook2_vo_note_targets( startup_msg_store ) diff --git a/vasl_templates/webapp/static/help/index.html b/vasl_templates/webapp/static/help/index.html index b73dd62..6d9ee52 100644 --- a/vasl_templates/webapp/static/help/index.html +++ b/vasl_templates/webapp/static/help/index.html @@ -374,6 +374,7 @@ The report also calculates "hotness", which is a measure of how hot you

Show Chapter H vehicle/ordnance notes as images

If you have set up the Chapter H vehicle/ordnance notes as HTML, it may not be possible to get the layout you want, since VASSAL's HTML engine is very old and doesn't support many HTML/CSS features. To work around this, this option tells VASL Templates to render the HTML itself (using a modern browser) and send it as an image to VASSAL, which is slower but gives better results. +

To optimize this process, the generated images are cached, and if you want to keep these cached images between sessions, configure VO_NOTES_IMAGE_CACHE_DIR in your site.cfg file. To pre-load this cache with all the available images, open http://localhost:5010/load-vo-notes-image-cache in a browser. diff --git a/vasl_templates/webapp/static/snippets.js b/vasl_templates/webapp/static/snippets.js index 4b2b790..bfbb1ca 100644 --- a/vasl_templates/webapp/static/snippets.js +++ b/vasl_templates/webapp/static/snippets.js @@ -577,17 +577,12 @@ function get_vo_note( vo_type, nat, key ) if ( !( key in gVehicleOrdnanceNotes[ vo_type ][ nat ] ) ) return null ; + // check if we have an image or HTML note var vo_note = gVehicleOrdnanceNotes[ vo_type ][ nat ][ key ] ; - // FUDGE! We need to detect between a full HTML note and an image-based one. - // This is not great, but it'll do... :-/ - var nat2 = nat ; - var pos = nat2.indexOf( "~" ) ; - if ( pos > 0 ) - nat2 = nat2.substring( 0, pos ) ; - if ( vo_note.substr( 0, nat2.length+1 ) === nat2+"/" ) - return make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ; + if ( vo_note.content !== undefined ) + return vo_note.content ; else - return vo_note ; + return make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ; } function get_ma_notes_keys( nat, vo_entries, vo_type ) diff --git a/vasl_templates/webapp/static/vo.js b/vasl_templates/webapp/static/vo.js index 97ae1aa..dffb334 100644 --- a/vasl_templates/webapp/static/vo.js +++ b/vasl_templates/webapp/static/vo.js @@ -234,23 +234,8 @@ function do_add_vo( vo_type, player_no, vo_entry, vo_image_id, elite, custom_cap "

", "" ] ; - var vo_note = get_vo_note( vo_type, nat, vo_note_key ) ; - var vo_note_image_url = null ; - if ( vo_note ) { - if ( is_landing_craft ) - vo_note_image_url = make_app_url( "/" + vo_type + "/landing-craft/note/" + vo_note_key.substring(3), true ) ; - else - vo_note_image_url = make_app_url( "/" + vo_type + "/" + nat + "/note/" + vo_note_key, true ) ; - } else { - // NOTE: Note numbers seem to be distinct across all Allied Minor or all Axis Minor vehicles/ordnance, - // so if we don't find a note in a given nationality's normal vehicles/ordnance, we can get away with - // just checking their corresponding common vehicles/ordnance. - if ( ["allied-minor","axis-minor"].indexOf( nat_type ) !== -1 ) { - vo_note = get_vo_note( vo_type, nat_type, vo_note_key ) ; - if ( vo_note ) - vo_note_image_url = make_app_url( "/" + vo_type + "/" + nat_type + "/note/" + vo_note_key, true ) ; - } - } + var rc = make_vo_note_image_url( vo_type, nat, vo_note_key ) ; + var vo_note_image_url = rc[0], vo_note = rc[1] ; if ( vo_note ) { var template_id = (vo_type === "vehicles") ? "ob_vehicle_note" : "ob_ordnance_note" ; if ( is_template_available( template_id ) ) { @@ -285,6 +270,35 @@ function do_add_vo( vo_type, player_no, vo_entry, vo_image_id, elite, custom_cap } ) ; } +function make_vo_note_image_url( vo_type, nat, key ) +{ + // generate the URL to get a vehicle/ordnance note image + var url = null ; + var vo_note = get_vo_note( vo_type, nat, key ) ; + if ( vo_note ) { + var is_landing_craft = key ? key.substring( 0, 3 ) === "LC " : null ; + if ( is_landing_craft ) + url = make_app_url( "/" + vo_type + "/landing-craft/note/" + key.substring(3), true ) ; + else + url = make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ; + } else { + // NOTE: Note numbers seem to be distinct across all Allied Minor or all Axis Minor vehicles/ordnance, + // so if we don't find a note in a given nationality's normal vehicles/ordnance, we can get away with + // just checking their corresponding common vehicles/ordnance. + var nat_type ; + if ( [ "allied-minor", "axis-minor" ].indexOf( nat ) !== -1 ) + nat_type = nat ; + else + nat_type = gTemplatePack.nationalities[ nat ].type ; + if ( [ "allied-minor", "axis-minor" ].indexOf( nat_type ) !== -1 ) { + vo_note = get_vo_note( vo_type, nat_type, key ) ; + if ( vo_note ) + url = make_app_url( "/" + vo_type + "/" + nat_type + "/note/" + key, true ) ; + } + } + return [ url, vo_note ] ; +} + function update_vo_sortable2_entry( $entry, vo_type, snippet_params ) { // initialize diff --git a/vasl_templates/webapp/templates/configure-vo-notes-image-cache.html b/vasl_templates/webapp/templates/configure-vo-notes-image-cache.html new file mode 100644 index 0000000..81d73ce --- /dev/null +++ b/vasl_templates/webapp/templates/configure-vo-notes-image-cache.html @@ -0,0 +1,7 @@ +If you have set up your vehicle/ordnance notes using HTML, and have configured "Show Chapter H vehicle/ordnance notes as images" in the Settings, you will find that scenarios can be slow to open in VASSAL, since converting each note to an image takes time. + +

This process can be optimized by configuring a directory to cache the generated images, and this page will pre-load the cache with all the available vehicle/ordnance notes. + +{%if NO_CACHE_DIR %} +

Configure VO_NOTES_IMAGE_CACHE_DIR in your site.cfg, restart the server, then reload this page. +{%endif%} diff --git a/vasl_templates/webapp/templates/load-vo-notes-image-cache.html b/vasl_templates/webapp/templates/load-vo-notes-image-cache.html new file mode 100644 index 0000000..b9b501d --- /dev/null +++ b/vasl_templates/webapp/templates/load-vo-notes-image-cache.html @@ -0,0 +1,136 @@ + + + + + + Load the vehicle/ordnance notes image cache + + + + + +

+ {%include "configure-vo-notes-image-cache.html" %} +

+

+
+
+
+ +
+ + + + + + + + + diff --git a/vasl_templates/webapp/templates/vo-notes-report.html b/vasl_templates/webapp/templates/vo-notes-report.html index abdbbc7..b8d68b2 100644 --- a/vasl_templates/webapp/templates/vo-notes-report.html +++ b/vasl_templates/webapp/templates/vo-notes-report.html @@ -148,7 +148,7 @@ function load_vo_notes( vo_entries ) var vo_note = vo_notes[ keys[i] ] ; buf.push( "", "", keys[i]+":", - "", vo_note.substr(vo_note.length-4) === ".png" ? vo_note : "(HTML content)" + "", vo_note.content !== undefined ? "(HTML content)" : vo_note.filename ) ; } buf.push( "" ) ; diff --git a/vasl_templates/webapp/tests/control_tests_servicer.py b/vasl_templates/webapp/tests/control_tests_servicer.py index bf98921..e9e8c40 100644 --- a/vasl_templates/webapp/tests/control_tests_servicer.py +++ b/vasl_templates/webapp/tests/control_tests_servicer.py @@ -158,6 +158,7 @@ class ControlTestsServicer( BaseControlTestsServicer ): #pylint: disable=too-man self.setAppConfigVal( SetAppConfigValRequest( key="DISABLE_LFA_HOTNESS_FADEIN", boolVal=True ), ctx ) self.deleteAppConfigVal( DeleteAppConfigValRequest( key="ASL_RULEBOOK2_BASE_URL" ), ctx ) self.deleteAppConfigVal( DeleteAppConfigValRequest( key="ALTERNATE_WEBAPP_BASE_URL" ), ctx ) + self.setAppConfigVal( SetAppConfigValRequest( key="VO_NOTES_IMAGE_CACHE_DIR", strVal="disabled" ), ctx ) # NOTE: The webapp has been reconfigured, but the client must reloaed the home page # with "?force-reinit=1", to force it to re-initialize with the new settings. @@ -495,6 +496,8 @@ class ControlTestsServicer( BaseControlTestsServicer ): #pylint: disable=too-man if request.HasField( val_type ): key, val = request.key, getattr(request,val_type) _logger.debug( "- Setting app config: %s = %s (%s)", key, str(val), type(val).__name__ ) + if val == "{{TEMP_DIR}}": + val = self._temp_dir.name self._webapp.config[ key ] = val return Empty() raise RuntimeError( "Can't find app config key." ) diff --git a/vasl_templates/webapp/tests/test_vo_notes.py b/vasl_templates/webapp/tests/test_vo_notes.py index dd15cc0..fdb5d9b 100644 --- a/vasl_templates/webapp/tests/test_vo_notes.py +++ b/vasl_templates/webapp/tests/test_vo_notes.py @@ -2,6 +2,7 @@ import os import shutil +import urllib.request import io import re @@ -214,6 +215,7 @@ def test_common_vo_notes2( webapp, webdriver ): # initialize webapp.control_tests.set_vo_notes_dir( "{TEST}" ) init_webapp( webapp, webdriver, scenario_persistence=1 ) + _enable_vo_no_notes_as_images( True ) # load the test scenario load_scenario( { @@ -224,13 +226,6 @@ def test_common_vo_notes2( webapp, webdriver ): ], } ) - # enable "show vehicle/ordnance notes as images" - select_menu_option( "user_settings" ) - elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" ) - assert not elem.is_selected() - elem.click() - click_dialog_button( "OK" ) - # check the snippets _check_vo_snippets( 1, "vehicles", [ ( "HTML note", "vehicles/greek/note/202" ), @@ -238,11 +233,6 @@ def test_common_vo_notes2( webapp, webdriver ): ] ) # restore "show vehicle/ordnance notes as images" - select_menu_option( "user_settings" ) - elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" ) - assert elem.is_selected() - elem.click() - click_dialog_button( "OK" ) # --------------------------------------------------------------------- @@ -345,6 +335,62 @@ def test_landing_craft_notes( webapp, webdriver ): # --------------------------------------------------------------------- +def test_vo_notes_image_cache( webapp, webdriver ): + """Test the vehicle/ordnance notes image cache.""" + + def init_test(): + # initialize the webapp + init_webapp( webapp, webdriver, scenario_persistence=1 ) + _enable_vo_no_notes_as_images( True ) + # load the test scenario + load_scenario( { + "PLAYER_1": "japanese", + "OB_VEHICLES_1": [ + { "name": "japanese vehicle" }, + ], + } ) + + # initialize + webapp.control_tests.set_vo_notes_dir( "{TEST}" ) + init_test() + + # get the vehicle note snippet + select_tab( "ob1" ) + elems = find_children( "#ob_vehicles-sortable_1 li" ) + assert len(elems) == 1 + btn = find_child( "img.snippet", elems[0] ) + btn.click() + expected = ( "japanese vehicle", "vehicles/japanese/note/2" ) + snippet = wait_for_clipboard( 2, expected, transform=_extract_vo_note ) + mo = re.search( r"", snippet ) + url = mo.group( 1 ) + + # get the vehicle note image (should be created) + resp = urllib.request.urlopen( url ) + assert not resp.headers.get( "X-WasCached" ) + image_data = resp.read() + + # get the vehicle note image (should be re-created) + resp = urllib.request.urlopen( url ) + assert not resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data + + # enable image caching + webapp.control_tests.set_app_config_val( "VO_NOTES_IMAGE_CACHE_DIR", "{{TEMP_DIR}}" ) + init_test() + + # get the vehicle note image (should be re-created) + resp = urllib.request.urlopen( url ) + assert not resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data + + # get the vehicle note image (should be cached) + resp = urllib.request.urlopen( url ) + assert resp.headers.get( "X-WasCached" ) + assert resp.read() == image_data + +# --------------------------------------------------------------------- + def test_update_ui( webapp, webdriver ): """Check that the UI is updated correctly for multi-applicable notes.""" @@ -676,3 +722,13 @@ def _extract_vo_note( clipboard ): return ( mo.group(1), mo.group(2) ) else: return clipboard + +def _enable_vo_no_notes_as_images( enable ): + """Enable/disable vehicle/ordnance notes as images.""" + select_menu_option( "user_settings" ) + elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" ) + if (elem.is_selected() and not enable) or (not elem.is_selected() and enable): + elem.click() + click_dialog_button( "OK" ) + else: + click_dialog_button( "Cancel" ) diff --git a/vasl_templates/webapp/vo_notes.py b/vasl_templates/webapp/vo_notes.py index 0e50005..a10cb0a 100644 --- a/vasl_templates/webapp/vo_notes.py +++ b/vasl_templates/webapp/vo_notes.py @@ -17,6 +17,8 @@ from vasl_templates.webapp.files import FileServer from vasl_templates.webapp.webdriver import WebDriver from vasl_templates.webapp.utils import read_text_file, resize_image_response, is_image_file, is_empty_file +_vo_notes_image_cache_dname = None + _asl_rulebook2_targets = None _asl_rulebook2_target_url_template = None @@ -137,7 +139,9 @@ def load_vo_notes( msg_store ): #pylint: disable=too-many-statements,too-many-lo # NOTE: We only do this if we don't already have an HTML version. if not vo_notes.get( vo_type2, {} ).get( nat2, {} ).get( key ): rel_path = os.path.relpath( fname, dname ) - vo_notes[vo_type2][nat2][key] = rel_path.replace( "\\", "/" ) + vo_notes[vo_type2][nat2][key] = { + "filename": rel_path.replace( "\\", "/" ) + } elif extn == ".html": @@ -171,10 +175,13 @@ def load_vo_notes( msg_store ): #pylint: disable=too-many-statements,too-many-lo if extn_id: key = "{}:{}".format( extn_id, key ) rel_path = os.path.relpath( os.path.split(fname)[0], dname ) - vo_notes[ vo_type2 ][ nat2 ][ key ] = _fixup_urls( - html_content, - "{{CHAPTER_H}}/" + rel_path.replace( "\\", "/" ) + "/" - ) + vo_notes[ vo_type2 ][ nat2 ][ key ] = { + "filename": fname, + "content": _fixup_urls( + html_content, + "{{CHAPTER_H}}/" + rel_path.replace( "\\", "/" ) + "/" + ) + } else: @@ -287,14 +294,14 @@ def get_vo_note( vo_type, nat, key ): abort( 404 ) # serve the file - if is_image_file( vo_note ): - resp = globvars.vo_notes_file_server.serve_file( vo_note, ignore_empty=True ) + if "content" not in vo_note: + resp = globvars.vo_notes_file_server.serve_file( vo_note["filename"], ignore_empty=True ) if not resp: abort( 404 ) default_scaling = app.config.get( "CHAPTER_H_IMAGE_SCALING", 100 ) return resize_image_response( resp, default_scaling=default_scaling ) else: - buf = _make_vo_note_html( vo_note ) + buf = _make_vo_note_html( vo_note["content"] ) if request.args.get( "f" ) == "html": # return the content as HTML return Response( buf, mimetype="text/html" ) @@ -307,14 +314,28 @@ def get_vo_note( vo_type, nat, key ): # a link that calls us here to generate the Chapter H content as an image, and if this 2nd request # gets handled in a different thread (which it certainly will, since the 1st request is still # in progress), we will deadlock waiting for the shared instance to become available. + cached_fname = _make_vo_note_cached_image_fname( vo_type, nat, key ) + if cached_fname and os.path.isfile( cached_fname ): + # we have a cached copy - compare the timestamps of the source HTML and the cached image + # NOTE: We should also check the HTML for any associated images, and check their timestamps, as well. + if os.path.getmtime( cached_fname ) >= os.path.getmtime( vo_note["filename"] ): + resp = send_file( cached_fname ) + resp.headers[ "X-WasCached" ] = 1 + return resp with WebDriver.get_instance( "vo_note" ) as webdriver: img = webdriver.get_snippet_screenshot( None, buf ) buf = io.BytesIO() img.save( buf, format="PNG" ) buf.seek( 0 ) + if cached_fname: + # save a copy of the generated image + os.makedirs( os.path.dirname( cached_fname ), exist_ok=True ) + with open( cached_fname, "wb" ) as fp: + fp.write( buf.read() ) + buf.seek( 0 ) return send_file( buf, mimetype="image/png" ) -def _make_vo_note_html( vo_note ): +def _make_vo_note_html( content ): """Generate the HTML for a vehicle/ordnance note.""" # initialize @@ -328,13 +349,31 @@ def _make_vo_note_html( vo_note ): globvars.template_pack.get( "css", {} ).get( "ob_vo_note", "" ), ] if any( css ): - vo_note = "\n\n\n\n{}".format( "\n".join(css), vo_note ) + content = "\n\n\n\n{}".format( "\n".join(css), content ) # update any parameters - vo_note = vo_note.replace( "{{CHAPTER_H}}", url_root+"/chapter-h" ) - vo_note = vo_note.replace( "{{IMAGES_BASE_URL}}", url_root+url_for("static",filename="images") ) + content = content.replace( "{{CHAPTER_H}}", url_root+"/chapter-h" ) + content = content.replace( "{{IMAGES_BASE_URL}}", url_root+url_for("static",filename="images") ) - return vo_note + return content + +def _make_vo_note_cached_image_fname( vo_type, nat, key ): + """Get the name of the cached vehicle/ordnance note image.""" + if not _vo_notes_image_cache_dname: + return None + return os.path.join( _vo_notes_image_cache_dname, vo_type, nat, key+".png" ) + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +@app.route( "/load-vo-notes-image-cache" ) +def load_vo_notes_image_cache(): + """Show the helper page to preload the v/o notes image cache.""" + dname = app.config.get( "VO_NOTES_IMAGE_CACHE_DIR" ) + if not dname: + return render_template( "configure-vo-notes-image-cache.html", + NO_CACHE_DIR = True + ) + return render_template( "load-vo-notes-image-cache.html" ) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -