Added a cache for vehicle/ordnance note images.

master
Pacman Ghost 3 years ago
parent 85a0085a8a
commit 089e43f31e
  1. 12
      vasl_templates/webapp/__init__.py
  2. 1
      vasl_templates/webapp/static/help/index.html
  3. 13
      vasl_templates/webapp/static/snippets.js
  4. 48
      vasl_templates/webapp/static/vo.js
  5. 7
      vasl_templates/webapp/templates/configure-vo-notes-image-cache.html
  6. 136
      vasl_templates/webapp/templates/load-vo-notes-image-cache.html
  7. 2
      vasl_templates/webapp/templates/vo-notes-report.html
  8. 3
      vasl_templates/webapp/tests/control_tests_servicer.py
  9. 80
      vasl_templates/webapp/tests/test_vo_notes.py
  10. 65
      vasl_templates/webapp/vo_notes.py

@ -84,6 +84,18 @@ def _init_webapp():
from vasl_templates.webapp.vo_notes import load_vo_notes #pylint: disable=cyclic-import
load_vo_notes( startup_msg_store )
# initialize the vehicle/ordnance notes image cache
from vasl_templates.webapp import vo_notes as webapp_vo_notes #pylint: disable=reimported
dname = app.config.get( "VO_NOTES_IMAGE_CACHE_DIR" )
if dname in ( "disable", "disabled" ):
webapp_vo_notes._vo_notes_image_cache_dname = None #pylint: disable=protected-access
elif dname:
webapp_vo_notes._vo_notes_image_cache_dname = dname #pylint: disable=protected-access
else:
webapp_vo_notes._vo_notes_image_cache_dname = os.path.join( #pylint: disable=protected-access
tempfile.gettempdir(), "vasl-templates", "vo-notes-image-cache"
)
# load integration data from asl-rulebook2
from vasl_templates.webapp.vo_notes import load_asl_rulebook2_vo_note_targets #pylint: disable=cyclic-import
load_asl_rulebook2_vo_note_targets( startup_msg_store )

@ -374,6 +374,7 @@ The report also calculates <em>"hotness"</em>, which is a measure of how hot you
<h4> Show Chapter H vehicle/ordnance notes as images </h4>
<p> If you have set up the <a href="#" onclick="select_tab('chapterh');">Chapter H vehicle/ordnance notes</a> as HTML, it may not be possible to get the layout you want, since VASSAL's HTML engine is very old and doesn't support many HTML/CSS features. To work around this, this option tells <em>VASL Templates</em> to render the HTML itself (using a modern browser) and send it as an image to VASSAL, which is slower but gives better results.
<p> To optimize this process, the generated images are cached, and if you want to keep these cached images between sessions, configure <tt>VO_NOTES_IMAGE_CACHE_DIR </tt> in your <tt>site.cfg</tt> file. To pre-load this cache with all the available images, open <tt>http://localhost:5010/load-vo-notes-image-cache</tt> in a browser.
</div>

@ -577,17 +577,12 @@ function get_vo_note( vo_type, nat, key )
if ( !( key in gVehicleOrdnanceNotes[ vo_type ][ nat ] ) )
return null ;
// check if we have an image or HTML note
var vo_note = gVehicleOrdnanceNotes[ vo_type ][ nat ][ key ] ;
// FUDGE! We need to detect between a full HTML note and an image-based one.
// This is not great, but it'll do... :-/
var nat2 = nat ;
var pos = nat2.indexOf( "~" ) ;
if ( pos > 0 )
nat2 = nat2.substring( 0, pos ) ;
if ( vo_note.substr( 0, nat2.length+1 ) === nat2+"/" )
return make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ;
if ( vo_note.content !== undefined )
return vo_note.content ;
else
return vo_note ;
return make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ;
}
function get_ma_notes_keys( nat, vo_entries, vo_type )

@ -234,23 +234,8 @@ function do_add_vo( vo_type, player_no, vo_entry, vo_image_id, elite, custom_cap
"<div class='vo-capabilities'></div>",
"</div>"
] ;
var vo_note = get_vo_note( vo_type, nat, vo_note_key ) ;
var vo_note_image_url = null ;
if ( vo_note ) {
if ( is_landing_craft )
vo_note_image_url = make_app_url( "/" + vo_type + "/landing-craft/note/" + vo_note_key.substring(3), true ) ;
else
vo_note_image_url = make_app_url( "/" + vo_type + "/" + nat + "/note/" + vo_note_key, true ) ;
} else {
// NOTE: Note numbers seem to be distinct across all Allied Minor or all Axis Minor vehicles/ordnance,
// so if we don't find a note in a given nationality's normal vehicles/ordnance, we can get away with
// just checking their corresponding common vehicles/ordnance.
if ( ["allied-minor","axis-minor"].indexOf( nat_type ) !== -1 ) {
vo_note = get_vo_note( vo_type, nat_type, vo_note_key ) ;
if ( vo_note )
vo_note_image_url = make_app_url( "/" + vo_type + "/" + nat_type + "/note/" + vo_note_key, true ) ;
}
}
var rc = make_vo_note_image_url( vo_type, nat, vo_note_key ) ;
var vo_note_image_url = rc[0], vo_note = rc[1] ;
if ( vo_note ) {
var template_id = (vo_type === "vehicles") ? "ob_vehicle_note" : "ob_ordnance_note" ;
if ( is_template_available( template_id ) ) {
@ -285,6 +270,35 @@ function do_add_vo( vo_type, player_no, vo_entry, vo_image_id, elite, custom_cap
} ) ;
}
function make_vo_note_image_url( vo_type, nat, key )
{
// generate the URL to get a vehicle/ordnance note image
var url = null ;
var vo_note = get_vo_note( vo_type, nat, key ) ;
if ( vo_note ) {
var is_landing_craft = key ? key.substring( 0, 3 ) === "LC " : null ;
if ( is_landing_craft )
url = make_app_url( "/" + vo_type + "/landing-craft/note/" + key.substring(3), true ) ;
else
url = make_app_url( "/" + vo_type + "/" + nat + "/note/" + key, true ) ;
} else {
// NOTE: Note numbers seem to be distinct across all Allied Minor or all Axis Minor vehicles/ordnance,
// so if we don't find a note in a given nationality's normal vehicles/ordnance, we can get away with
// just checking their corresponding common vehicles/ordnance.
var nat_type ;
if ( [ "allied-minor", "axis-minor" ].indexOf( nat ) !== -1 )
nat_type = nat ;
else
nat_type = gTemplatePack.nationalities[ nat ].type ;
if ( [ "allied-minor", "axis-minor" ].indexOf( nat_type ) !== -1 ) {
vo_note = get_vo_note( vo_type, nat_type, key ) ;
if ( vo_note )
url = make_app_url( "/" + vo_type + "/" + nat_type + "/note/" + key, true ) ;
}
}
return [ url, vo_note ] ;
}
function update_vo_sortable2_entry( $entry, vo_type, snippet_params )
{
// initialize

@ -0,0 +1,7 @@
If you have set up your vehicle/ordnance notes using HTML, and have configured <em>"Show Chapter H vehicle/ordnance notes as images"</em> in the Settings, you will find that scenarios can be slow to open in VASSAL, since converting each note to an image takes time.
<p> This process can be optimized by configuring a directory to cache the generated images, and this page will pre-load the cache with all the available vehicle/ordnance notes.
{%if NO_CACHE_DIR %}
<p> Configure <tt>VO_NOTES_IMAGE_CACHE_DIR</tt> in your <tt>site.cfg</tt>, restart the server, then reload this page.
{%endif%}

@ -0,0 +1,136 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title> Load the vehicle/ordnance notes image cache </title>
<link rel="stylesheet" type="text/css" href="{{url_for('static',filename='jquery-ui/jquery-ui.min.css')}}" />
<style>
#progress { position: relative ; margin-bottom: 5px ; }
#progress .caption { position: absolute ; top: 7px ; left: 49% ; color: #666 ; }
#status { color: #333 ; }
img#vo-note { max-width: 95% ; border: 1px dotted #666 ; border-radius: 5px ; padding: 8px ; display: none ; }
</style>
</head>
<body>
<div style="width:49%;float:left;">
{%include "configure-vo-notes-image-cache.html" %}
<p> <button id="start"> Go </button>
<div id="progress"> <div class="caption"></div> </div>
<div id="status"></div>
</div>
<div style="width:49%;float:right;">
<img id="vo-note">
</div>
</body>
<script src="{{url_for('static',filename='jquery/jquery-3.3.1.min.js')}}"></script>
<script src="{{url_for('static',filename='jquery-ui/jquery-ui.min.js')}}"></script>
<script src="{{url_for('static',filename='snippets.js')}}"></script>
<script src="{{url_for('static',filename='vo.js')}}"></script>
<script src="{{url_for('static',filename='utils.js')}}"></script>
<script>
gAppConfig = {} ;
gTemplatePack = {} ;
gVehicleOrdnanceNotes = {} ;
gTotalEntries = 0 ;
$( document ).ready( function () {
// initialize
var startup = 3 ;
$( "#start" ).attr( "disabled", true ).on( "click", main ) ;
// get the template pack
$.getJSON( "{{url_for( 'get_template_pack' )}}", function( resp ) {
gTemplatePack = resp ;
-- startup ;
} ).fail( function( xhr, status, errorMsg ) {
alert( "Can't get the nationalities:\n\n" + errorMsg ) ;
} ) ;
// get the vehicle/ordnance notes
function get_vo_notes( vo_type, url ) {
$.getJSON( url, function( resp ) {
gVehicleOrdnanceNotes[ vo_type ] = resp ;
if ( --startup == 0 )
$( "#start" ).attr( "disabled", false ) ;
} ).fail( function( xhr, status, errorMsg ) {
alert( "Can't get the " + vo_type + " notes:\n\n" + errorMsg ) ;
} ) ;
}
get_vo_notes( "vehicles", "{{url_for('get_vehicle_notes')}}" ) ;
get_vo_notes( "ordnance", "{{url_for('get_ordnance_notes')}}" ) ;
} ) ;
function main() {
// generate all the requests we need to make
var requests = [] ;
[ "vehicles", "ordnance" ].forEach( function( vo_type ) {
for ( var nat in gVehicleOrdnanceNotes[ vo_type ] ) {
for ( var key in gVehicleOrdnanceNotes[ vo_type ][ nat ] ) {
if ( key == "multi-applicable" )
continue ;
var rc = make_vo_note_image_url( vo_type, nat, key ) ;
var url = rc[0] ;
if ( ! url )
continue ;
var nat_info = gTemplatePack.nationalities[ nat ] ;
var nat_name = nat_info ? nat_info.display_name : nat ;
var caption = vo_type + ": " + nat_name + " #" + key ;
requests.push( [ url, caption ] ) ;
}
}
} ) ;
var nRequests = requests.length ;
if ( nRequests == 0 ) {
alert( "No vehicle/ordnance notes were found." ) ;
return ;
}
// initialize
$( "#start" ).hide() ;
var $img = $( "img#vo-note" ).on( "load", function() {
// the vehicle/ordnance note finished loading - request the next one
$(this).show() ;
processNextRequest() ;
} ) ;
var $status = $( "#status" ) ;
var $progress = $( "#progress" ).progressbar( {
change: function( ) {
var val = $progress.progressbar( "value" ) ;
$progress.find( ".caption" ).text( val + "%" ) ;
},
complete: function() {
$progress.find( ".caption" ).text( "" ) ;
},
} ) ;
// fetch each vehicle/ordnance note image
function processNextRequest() {
if ( requests.length == 0 ) {
$progress.progressbar( { value: 100 } ) ;
$progress.find( ".caption" ).text( "100%" ) ;
$status.text( "All done." ) ;
return ;
}
var req = requests.shift() ;
var val = Math.floor( 100 * requests.length / nRequests ) ;
$progress.progressbar( {
value: Math.min( 100-val, 99 )
} ) ;
$status.text( "Generating " + req[1] ) ;
// NOTE: We show the generated the image to the user for visual feedback, which will trigger
// a "load" event when it has finished loading, which will invoke processNextRequest() to get
// the next image, until they're all done.
$img.attr( "src", req[0] ) ;
}
processNextRequest() ;
}
</script>

@ -148,7 +148,7 @@ function load_vo_notes( vo_entries )
var vo_note = vo_notes[ keys[i] ] ;
buf.push( "<tr>",
"<td class='key'>", keys[i]+":",
"<td>", vo_note.substr(vo_note.length-4) === ".png" ? vo_note : "(HTML content)"
"<td>", vo_note.content !== undefined ? "(HTML content)" : vo_note.filename
) ;
}
buf.push( "</table>" ) ;

@ -158,6 +158,7 @@ class ControlTestsServicer( BaseControlTestsServicer ): #pylint: disable=too-man
self.setAppConfigVal( SetAppConfigValRequest( key="DISABLE_LFA_HOTNESS_FADEIN", boolVal=True ), ctx )
self.deleteAppConfigVal( DeleteAppConfigValRequest( key="ASL_RULEBOOK2_BASE_URL" ), ctx )
self.deleteAppConfigVal( DeleteAppConfigValRequest( key="ALTERNATE_WEBAPP_BASE_URL" ), ctx )
self.setAppConfigVal( SetAppConfigValRequest( key="VO_NOTES_IMAGE_CACHE_DIR", strVal="disabled" ), ctx )
# NOTE: The webapp has been reconfigured, but the client must reloaed the home page
# with "?force-reinit=1", to force it to re-initialize with the new settings.
@ -495,6 +496,8 @@ class ControlTestsServicer( BaseControlTestsServicer ): #pylint: disable=too-man
if request.HasField( val_type ):
key, val = request.key, getattr(request,val_type)
_logger.debug( "- Setting app config: %s = %s (%s)", key, str(val), type(val).__name__ )
if val == "{{TEMP_DIR}}":
val = self._temp_dir.name
self._webapp.config[ key ] = val
return Empty()
raise RuntimeError( "Can't find app config key." )

@ -2,6 +2,7 @@
import os
import shutil
import urllib.request
import io
import re
@ -214,6 +215,7 @@ def test_common_vo_notes2( webapp, webdriver ):
# initialize
webapp.control_tests.set_vo_notes_dir( "{TEST}" )
init_webapp( webapp, webdriver, scenario_persistence=1 )
_enable_vo_no_notes_as_images( True )
# load the test scenario
load_scenario( {
@ -224,13 +226,6 @@ def test_common_vo_notes2( webapp, webdriver ):
],
} )
# enable "show vehicle/ordnance notes as images"
select_menu_option( "user_settings" )
elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" )
assert not elem.is_selected()
elem.click()
click_dialog_button( "OK" )
# check the snippets
_check_vo_snippets( 1, "vehicles", [
( "HTML note", "vehicles/greek/note/202" ),
@ -238,11 +233,6 @@ def test_common_vo_notes2( webapp, webdriver ):
] )
# restore "show vehicle/ordnance notes as images"
select_menu_option( "user_settings" )
elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" )
assert elem.is_selected()
elem.click()
click_dialog_button( "OK" )
# ---------------------------------------------------------------------
@ -345,6 +335,62 @@ def test_landing_craft_notes( webapp, webdriver ):
# ---------------------------------------------------------------------
def test_vo_notes_image_cache( webapp, webdriver ):
"""Test the vehicle/ordnance notes image cache."""
def init_test():
# initialize the webapp
init_webapp( webapp, webdriver, scenario_persistence=1 )
_enable_vo_no_notes_as_images( True )
# load the test scenario
load_scenario( {
"PLAYER_1": "japanese",
"OB_VEHICLES_1": [
{ "name": "japanese vehicle" },
],
} )
# initialize
webapp.control_tests.set_vo_notes_dir( "{TEST}" )
init_test()
# get the vehicle note snippet
select_tab( "ob1" )
elems = find_children( "#ob_vehicles-sortable_1 li" )
assert len(elems) == 1
btn = find_child( "img.snippet", elems[0] )
btn.click()
expected = ( "japanese vehicle", "vehicles/japanese/note/2" )
snippet = wait_for_clipboard( 2, expected, transform=_extract_vo_note )
mo = re.search( r"<img src=\"(.+?)\">", snippet )
url = mo.group( 1 )
# get the vehicle note image (should be created)
resp = urllib.request.urlopen( url )
assert not resp.headers.get( "X-WasCached" )
image_data = resp.read()
# get the vehicle note image (should be re-created)
resp = urllib.request.urlopen( url )
assert not resp.headers.get( "X-WasCached" )
assert resp.read() == image_data
# enable image caching
webapp.control_tests.set_app_config_val( "VO_NOTES_IMAGE_CACHE_DIR", "{{TEMP_DIR}}" )
init_test()
# get the vehicle note image (should be re-created)
resp = urllib.request.urlopen( url )
assert not resp.headers.get( "X-WasCached" )
assert resp.read() == image_data
# get the vehicle note image (should be cached)
resp = urllib.request.urlopen( url )
assert resp.headers.get( "X-WasCached" )
assert resp.read() == image_data
# ---------------------------------------------------------------------
def test_update_ui( webapp, webdriver ):
"""Check that the UI is updated correctly for multi-applicable notes."""
@ -676,3 +722,13 @@ def _extract_vo_note( clipboard ):
return ( mo.group(1), mo.group(2) )
else:
return clipboard
def _enable_vo_no_notes_as_images( enable ):
"""Enable/disable vehicle/ordnance notes as images."""
select_menu_option( "user_settings" )
elem = find_child( ".ui-dialog.user-settings input[name='vo-notes-as-images']" )
if (elem.is_selected() and not enable) or (not elem.is_selected() and enable):
elem.click()
click_dialog_button( "OK" )
else:
click_dialog_button( "Cancel" )

@ -17,6 +17,8 @@ from vasl_templates.webapp.files import FileServer
from vasl_templates.webapp.webdriver import WebDriver
from vasl_templates.webapp.utils import read_text_file, resize_image_response, is_image_file, is_empty_file
_vo_notes_image_cache_dname = None
_asl_rulebook2_targets = None
_asl_rulebook2_target_url_template = None
@ -137,7 +139,9 @@ def load_vo_notes( msg_store ): #pylint: disable=too-many-statements,too-many-lo
# NOTE: We only do this if we don't already have an HTML version.
if not vo_notes.get( vo_type2, {} ).get( nat2, {} ).get( key ):
rel_path = os.path.relpath( fname, dname )
vo_notes[vo_type2][nat2][key] = rel_path.replace( "\\", "/" )
vo_notes[vo_type2][nat2][key] = {
"filename": rel_path.replace( "\\", "/" )
}
elif extn == ".html":
@ -171,10 +175,13 @@ def load_vo_notes( msg_store ): #pylint: disable=too-many-statements,too-many-lo
if extn_id:
key = "{}:{}".format( extn_id, key )
rel_path = os.path.relpath( os.path.split(fname)[0], dname )
vo_notes[ vo_type2 ][ nat2 ][ key ] = _fixup_urls(
html_content,
"{{CHAPTER_H}}/" + rel_path.replace( "\\", "/" ) + "/"
)
vo_notes[ vo_type2 ][ nat2 ][ key ] = {
"filename": fname,
"content": _fixup_urls(
html_content,
"{{CHAPTER_H}}/" + rel_path.replace( "\\", "/" ) + "/"
)
}
else:
@ -287,14 +294,14 @@ def get_vo_note( vo_type, nat, key ):
abort( 404 )
# serve the file
if is_image_file( vo_note ):
resp = globvars.vo_notes_file_server.serve_file( vo_note, ignore_empty=True )
if "content" not in vo_note:
resp = globvars.vo_notes_file_server.serve_file( vo_note["filename"], ignore_empty=True )
if not resp:
abort( 404 )
default_scaling = app.config.get( "CHAPTER_H_IMAGE_SCALING", 100 )
return resize_image_response( resp, default_scaling=default_scaling )
else:
buf = _make_vo_note_html( vo_note )
buf = _make_vo_note_html( vo_note["content"] )
if request.args.get( "f" ) == "html":
# return the content as HTML
return Response( buf, mimetype="text/html" )
@ -307,14 +314,28 @@ def get_vo_note( vo_type, nat, key ):
# a link that calls us here to generate the Chapter H content as an image, and if this 2nd request
# gets handled in a different thread (which it certainly will, since the 1st request is still
# in progress), we will deadlock waiting for the shared instance to become available.
cached_fname = _make_vo_note_cached_image_fname( vo_type, nat, key )
if cached_fname and os.path.isfile( cached_fname ):
# we have a cached copy - compare the timestamps of the source HTML and the cached image
# NOTE: We should also check the HTML for any associated images, and check their timestamps, as well.
if os.path.getmtime( cached_fname ) >= os.path.getmtime( vo_note["filename"] ):
resp = send_file( cached_fname )
resp.headers[ "X-WasCached" ] = 1
return resp
with WebDriver.get_instance( "vo_note" ) as webdriver:
img = webdriver.get_snippet_screenshot( None, buf )
buf = io.BytesIO()
img.save( buf, format="PNG" )
buf.seek( 0 )
if cached_fname:
# save a copy of the generated image
os.makedirs( os.path.dirname( cached_fname ), exist_ok=True )
with open( cached_fname, "wb" ) as fp:
fp.write( buf.read() )
buf.seek( 0 )
return send_file( buf, mimetype="image/png" )
def _make_vo_note_html( vo_note ):
def _make_vo_note_html( content ):
"""Generate the HTML for a vehicle/ordnance note."""
# initialize
@ -328,13 +349,31 @@ def _make_vo_note_html( vo_note ):
globvars.template_pack.get( "css", {} ).get( "ob_vo_note", "" ),
]
if any( css ):
vo_note = "<head>\n<style>\n{}\n</style>\n</head>\n\n{}".format( "\n".join(css), vo_note )
content = "<head>\n<style>\n{}\n</style>\n</head>\n\n{}".format( "\n".join(css), content )
# update any parameters
vo_note = vo_note.replace( "{{CHAPTER_H}}", url_root+"/chapter-h" )
vo_note = vo_note.replace( "{{IMAGES_BASE_URL}}", url_root+url_for("static",filename="images") )
content = content.replace( "{{CHAPTER_H}}", url_root+"/chapter-h" )
content = content.replace( "{{IMAGES_BASE_URL}}", url_root+url_for("static",filename="images") )
return vo_note
return content
def _make_vo_note_cached_image_fname( vo_type, nat, key ):
"""Get the name of the cached vehicle/ordnance note image."""
if not _vo_notes_image_cache_dname:
return None
return os.path.join( _vo_notes_image_cache_dname, vo_type, nat, key+".png" )
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@app.route( "/load-vo-notes-image-cache" )
def load_vo_notes_image_cache():
"""Show the helper page to preload the v/o notes image cache."""
dname = app.config.get( "VO_NOTES_IMAGE_CACHE_DIR" )
if not dname:
return render_template( "configure-vo-notes-image-cache.html",
NO_CACHE_DIR = True
)
return render_template( "load-vo-notes-image-cache.html" )
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Loading…
Cancel
Save