parent
fdd027cb2b
commit
fee9f49f1c
@ -0,0 +1,19 @@ |
|||||||
|
""" Helper utilities. """ |
||||||
|
|
||||||
|
import os |
||||||
|
|
||||||
|
from asl_rulebook2.tests import pytest_options |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
def for_each_easlrb_version( func ): |
||||||
|
"""Run tests for each version of the eASLRB.""" |
||||||
|
assert pytest_options.easlrb_path |
||||||
|
base_dir = pytest_options.easlrb_path |
||||||
|
ncalls = 0 |
||||||
|
for name in os.listdir( base_dir ): |
||||||
|
dname = os.path.join( base_dir, name ) |
||||||
|
if os.path.isfile( os.path.join( dname, "eASLRB.pdf" ) ): |
||||||
|
func( dname ) |
||||||
|
ncalls += 1 |
||||||
|
assert ncalls > 0 |
@ -0,0 +1,214 @@ |
|||||||
|
""" Analyze the MMP eASLRB PDF and prepare the data files. """ |
||||||
|
|
||||||
|
import threading |
||||||
|
import zipfile |
||||||
|
import io |
||||||
|
import time |
||||||
|
import base64 |
||||||
|
import traceback |
||||||
|
import logging |
||||||
|
|
||||||
|
from flask import request, send_file, abort, url_for |
||||||
|
|
||||||
|
from asl_rulebook2.extract.all import ExtractAll |
||||||
|
from asl_rulebook2.bin.prepare_pdf import prepare_pdf |
||||||
|
from asl_rulebook2.bin.fixup_mmp_pdf import fixup_mmp_pdf |
||||||
|
from asl_rulebook2.pdf import PdfDoc |
||||||
|
from asl_rulebook2.utils import TempFile |
||||||
|
from asl_rulebook2.webapp import app, globvars |
||||||
|
from asl_rulebook2.webapp.utils import get_gs_path |
||||||
|
|
||||||
|
_zip_data_download = None |
||||||
|
|
||||||
|
_logger = logging.getLogger( "prepare" ) |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
@app.route( "/prepare", methods=["POST"] ) |
||||||
|
def prepare_data_files(): |
||||||
|
"""Prepare the data files.""" |
||||||
|
|
||||||
|
# initialize |
||||||
|
args = dict( request.json ) |
||||||
|
download_url = url_for( "download_prepared_data" ) |
||||||
|
|
||||||
|
# initialize the socketio server |
||||||
|
sio = globvars.socketio_server |
||||||
|
if not sio: |
||||||
|
raise RuntimeError( "The socketio server has not been started." ) |
||||||
|
@sio.on( "start" ) |
||||||
|
def on_start( data ): #pylint: disable=unused-variable,unused-argument |
||||||
|
# start the worker thread that prepares the data files |
||||||
|
# NOTE: We don't do this when the POST request comes in, but wait until the client |
||||||
|
# tells us it's ready (otherwise, it might miss the first event or two). |
||||||
|
def worker(): |
||||||
|
try: |
||||||
|
_do_prepare_data_files( args, download_url ) |
||||||
|
except Exception as ex: #pylint: disable=broad-except |
||||||
|
_logger.error( "PREPARE ERROR: %s\n%s", ex, traceback.format_exc() ) |
||||||
|
globvars.socketio_server.emit( "error", str(ex) ) |
||||||
|
threading.Thread( target=worker, daemon=True ).start() |
||||||
|
|
||||||
|
return "ok" |
||||||
|
|
||||||
|
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
||||||
|
|
||||||
|
def _do_prepare_data_files( args, download_url ): |
||||||
|
|
||||||
|
# initialize |
||||||
|
sio = globvars.socketio_server |
||||||
|
pdf_data = args.get( "pdfData" ) |
||||||
|
if not pdf_data: |
||||||
|
# no data was sent - this is a test of logging progress messages. |
||||||
|
del args["pdfData"] |
||||||
|
_test_progress( **args ) |
||||||
|
return |
||||||
|
pdf_data = base64.b64decode( pdf_data ) |
||||||
|
|
||||||
|
def on_done( zip_data ): |
||||||
|
global _zip_data_download |
||||||
|
_zip_data_download = zip_data |
||||||
|
sio.emit( "done", download_url ) |
||||||
|
|
||||||
|
# check if we should just return a pre-prepared ZIP file (for testing porpoises) |
||||||
|
fname = app.config.get( "PREPARED_ZIP" ) |
||||||
|
if fname: |
||||||
|
with open( fname, "rb" ) as fp: |
||||||
|
on_done( fp.read() ) |
||||||
|
return |
||||||
|
|
||||||
|
with TempFile() as input_file, TempFile() as prepared_file: |
||||||
|
|
||||||
|
# save the PDF file data |
||||||
|
input_file.write( pdf_data ) |
||||||
|
input_file.close( delete=False ) |
||||||
|
_logger.info( "Saved PDF file (#bytes=%d): %s", len(pdf_data), input_file.name ) |
||||||
|
|
||||||
|
# initialize logging |
||||||
|
msg_types = set() |
||||||
|
def log_msg( msg_type, msg ): |
||||||
|
msg = msg.lstrip() |
||||||
|
if msg_type == "status": |
||||||
|
_logger.info( "[STATUS]: %s", msg ) |
||||||
|
elif msg_type == "warning": |
||||||
|
_logger.warning( "[WARNING]: %s", msg ) |
||||||
|
elif msg_type == "error": |
||||||
|
_logger.error( "[ERROR]: %s", msg ) |
||||||
|
else: |
||||||
|
_logger.debug( "[%s] %s", msg_type, msg ) |
||||||
|
if msg.startswith( "- " ): |
||||||
|
msg = msg[2:] |
||||||
|
sio.emit( msg_type, msg ) |
||||||
|
msg_types.add( msg_type ) |
||||||
|
|
||||||
|
# NOTE: The plan was to allow the user to change the default parameters in the UI, |
||||||
|
# but this can be done (ahem) later. For now, if they really need to change something, |
||||||
|
# they can prepare the data files from the command-line. |
||||||
|
args = [] |
||||||
|
|
||||||
|
# extract everything we need from the PDF |
||||||
|
log_msg( "status", "Opening the PDF..." ) |
||||||
|
extract = ExtractAll( args, log_msg ) |
||||||
|
with PdfDoc( input_file.name ) as pdf: |
||||||
|
extract.extract_all( pdf ) |
||||||
|
index_buf = io.StringIO() |
||||||
|
extract.extract_index.save_as_json( index_buf ) |
||||||
|
targets_buf, chapters_buf, footnotes_buf = io.StringIO(), io.StringIO(), io.StringIO() |
||||||
|
extract.extract_content.save_as_json( targets_buf, chapters_buf, footnotes_buf ) |
||||||
|
file_data = { |
||||||
|
"index": index_buf.getvalue(), |
||||||
|
"targets": targets_buf.getvalue(), |
||||||
|
"chapters": chapters_buf.getvalue(), |
||||||
|
"footnotes": footnotes_buf.getvalue(), |
||||||
|
} |
||||||
|
|
||||||
|
# prepare the PDF |
||||||
|
gs_path = get_gs_path() |
||||||
|
if not gs_path: |
||||||
|
raise RuntimeError( "Ghostscript is not available." ) |
||||||
|
with TempFile( mode="w", encoding="utf-8" ) as targets_file: |
||||||
|
log_msg( "status", "Preparing the final PDF..." ) |
||||||
|
# save the extracted targets |
||||||
|
targets_file.temp_file.write( file_data["targets"] ) |
||||||
|
targets_file.close( delete=False ) |
||||||
|
# prepare the PDF |
||||||
|
prepared_file.close( delete=False ) |
||||||
|
prepare_pdf( input_file.name, |
||||||
|
"ASL Rulebook", |
||||||
|
targets_file.name, 5, |
||||||
|
prepared_file.name, "ebook", |
||||||
|
gs_path, |
||||||
|
log_msg |
||||||
|
) |
||||||
|
|
||||||
|
# fixup the PDF |
||||||
|
with TempFile() as fixedup_file: |
||||||
|
log_msg( "status", "Fixing up the final PDF..." ) |
||||||
|
fixedup_file.close( delete=False ) |
||||||
|
fixup_mmp_pdf( prepared_file.name, |
||||||
|
fixedup_file.name, |
||||||
|
True, True, |
||||||
|
log_msg |
||||||
|
) |
||||||
|
# read the final PDF data |
||||||
|
with open( fixedup_file.name, "rb" ) as fp: |
||||||
|
pdf_data = fp.read() |
||||||
|
|
||||||
|
# prepare the ZIP for the user to download |
||||||
|
log_msg( "status", "Preparing the download ZIP..." ) |
||||||
|
zip_data = io.BytesIO() |
||||||
|
with zipfile.ZipFile( zip_data, "w", zipfile.ZIP_DEFLATED ) as zip_file: |
||||||
|
fname_stem = "ASL Rulebook" |
||||||
|
zip_file.writestr( fname_stem+".pdf", pdf_data ) |
||||||
|
for key in file_data: |
||||||
|
fname = "{}.{}".format( fname_stem, key ) |
||||||
|
zip_file.writestr( fname, file_data[key] ) |
||||||
|
zip_data = zip_data.getvalue() |
||||||
|
|
||||||
|
# notify the front-end that we're done |
||||||
|
on_done( zip_data ) |
||||||
|
_logger.debug( "Message types seen: %s", |
||||||
|
" ; ".join( sorted( str(mt) for mt in msg_types ) ) |
||||||
|
) |
||||||
|
|
||||||
|
# NOTE: We don't bother shutting down the socketio server, since the user |
||||||
|
# has to restart the server, using the newly-prepared data files. |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
@app.route( "/prepare/download" ) |
||||||
|
def download_prepared_data(): |
||||||
|
"""Download the prepared data ZIP file.""" |
||||||
|
if not _zip_data_download: |
||||||
|
abort( 404 ) |
||||||
|
return send_file( |
||||||
|
io.BytesIO( _zip_data_download ), |
||||||
|
as_attachment=True, attachment_filename="asl-rulebook2.zip" |
||||||
|
) |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
def _test_progress( npasses=100, status=10, warnings=None, errors=None, delay=0.1 ): |
||||||
|
"""Test progress messages.""" |
||||||
|
|
||||||
|
# initialize |
||||||
|
warnings = [ int(w) for w in warnings.split(",") ] if warnings else [] |
||||||
|
errors = [ int(e) for e in errors.split(",") ] if errors else [] |
||||||
|
|
||||||
|
# generate progress messages |
||||||
|
sio = globvars.socketio_server |
||||||
|
status_no = 0 |
||||||
|
for i in range( int(npasses) ): |
||||||
|
# check if we should start a new status block |
||||||
|
if i % status == 0: |
||||||
|
status_no += 1 |
||||||
|
sio.emit( "status", "Status #{}".format( status_no ) ) |
||||||
|
# issue the next progress message |
||||||
|
if 1+i in warnings: |
||||||
|
sio.emit( "warning", "Progress {}: warning".format( 1+i ) ) |
||||||
|
if 1+i in errors: |
||||||
|
sio.emit( "error", "Progress {}: error".format( 1+i ) ) |
||||||
|
else: |
||||||
|
sio.emit( "progress", "Progress {}.".format( 1+i ) ) |
||||||
|
time.sleep( float( delay ) ) |
||||||
|
sio.emit( "done" ) |
@ -0,0 +1,37 @@ |
|||||||
|
p { margin: 5px 0 ; } |
||||||
|
code { display: block ; margin: 5px 0 5px 20px ; } |
||||||
|
.info { |
||||||
|
margin-top: 10px ; min-height: 25px ; |
||||||
|
padding-left: 30px ; background: no-repeat url(../images/info.png) ; |
||||||
|
font-size: 80% ; font-style: italic ; color: #444 ; |
||||||
|
} |
||||||
|
|
||||||
|
#prepare-app { height: 100% ; display: flex ; } |
||||||
|
#header { margin-bottom: 5px ; } |
||||||
|
#main { width: 100% ; margin: 10px ; display: flex ; flex-direction: column ; } |
||||||
|
|
||||||
|
#fatal-error { margin-bottom: 10px ; font-size: 120% ; font-weight: bold ; } |
||||||
|
|
||||||
|
#upload-panel { align-self: start ; border: 1px solid black ; border-radius: 5px ; padding: 10px ; } |
||||||
|
#upload-panel button { height: 70px ; margin-right: 10px ; } |
||||||
|
#upload-panel button img { margin-top: 3px ; height: 60px ; } |
||||||
|
|
||||||
|
#progress-panel { |
||||||
|
flex-grow: 1 ; overflow-y: auto ; |
||||||
|
border: 1px solid black ; border-radius: 5px ; padding: 10px ; |
||||||
|
font-family: monospace ; font-size: 90% ; |
||||||
|
} |
||||||
|
#progress-panel .progress { font-style: italic ; } |
||||||
|
#progress-panel .status { margin: 5px 0 ; } |
||||||
|
#progress-panel .status:first-of-type { margin-top: 0 ; } |
||||||
|
#progress-panel .status table { margin-left: 2px ; } |
||||||
|
#progress-panel .status table td { vertical-align: top ; } |
||||||
|
#progress-panel .status img.icon { height: 15px ; margin: 1px 3px 0 0 ; } |
||||||
|
|
||||||
|
#download-panel { |
||||||
|
position: fixed ; bottom: 18px ; right: 18px ; width: 75% ; |
||||||
|
border: 1px solid black ; border-radius: 5px ; background: white ; |
||||||
|
padding: 10px ; |
||||||
|
} |
||||||
|
#download-panel button { height: 40px ; margin-right: 10px ; padding: 5px ; } |
||||||
|
#download-panel button img { height: 30px ; } |
After Width: | Height: | Size: 2.5 KiB |
After Width: | Height: | Size: 12 KiB |
After Width: | Height: | Size: 1.2 KiB |
After Width: | Height: | Size: 1.1 KiB |
@ -0,0 +1,364 @@ |
|||||||
|
// create the main application
|
||||||
|
export const gPrepareApp = Vue.createApp( { //eslint-disable-line no-undef
|
||||||
|
template: "<prepare-app />", |
||||||
|
} ) ; |
||||||
|
$(document).ready( () => { |
||||||
|
gPrepareApp.mount( "#prepare-app" ) ; |
||||||
|
} ) ; |
||||||
|
|
||||||
|
// parse any URL parameters
|
||||||
|
let gUrlParams = new URLSearchParams( window.location.search.substring(1) ) ; |
||||||
|
|
||||||
|
let gProgressPanel = null ; |
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
|
||||||
|
gPrepareApp.component( "prepare-app", { |
||||||
|
|
||||||
|
data() { return { |
||||||
|
isLoaded: false, |
||||||
|
isProcessing: false, |
||||||
|
downloadUrl: null, |
||||||
|
fatalErrorMsg: gHaveGhostscript ? null : "Ghostscript is not available.", //eslint-disable-line no-undef
|
||||||
|
fatalErrorIconUrl: makeImageUrl( "error.png" ), |
||||||
|
} ; }, |
||||||
|
|
||||||
|
template: ` |
||||||
|
<div id="main"> |
||||||
|
<div id="header"> |
||||||
|
No data directory has been configured. |
||||||
|
<p> If you haven't used this program before, a few things need to be prepared first. |
||||||
|
It will take around 10-15 minutes. </p> |
||||||
|
</div> |
||||||
|
<div v-show=fatalErrorMsg id="fatal-error" > |
||||||
|
<img :src=fatalErrorIconUrl style="float:left;margin-right:5px;" /> |
||||||
|
{{fatalErrorMsg}} |
||||||
|
</div> |
||||||
|
<upload-panel v-show="!fatalErrorMsg &&!isProcessing" @file-selected=onFileSelected /> |
||||||
|
<progress-panel v-show=isProcessing @done=onDone @fatal=onFatalError ref=progressPanel /> |
||||||
|
<download-panel v-show=downloadUrl :downloadUrl=downloadUrl ref=downloadPanel /> |
||||||
|
<textarea id="testing-zip-data" style="display:none;" /> |
||||||
|
<div v-if=isLoaded id="_prepareapp-loaded_" /> |
||||||
|
</div>`, |
||||||
|
|
||||||
|
mounted() { |
||||||
|
// initialize the UI
|
||||||
|
$( "button" ).button() ; |
||||||
|
this.isLoaded = true ; |
||||||
|
}, |
||||||
|
|
||||||
|
methods: { |
||||||
|
|
||||||
|
onFileSelected( file ) { |
||||||
|
this.isProcessing = true ; |
||||||
|
if ( ! file ) { |
||||||
|
// this is a test of progress logging
|
||||||
|
this.uploadPdfData( null ) ; |
||||||
|
return ; |
||||||
|
} |
||||||
|
if ( typeof file == "string" ) { |
||||||
|
// this is PDF file data given to us by the test suite - just return it as is
|
||||||
|
this.uploadPdfData( file ) ; |
||||||
|
return ; |
||||||
|
} |
||||||
|
this.$nextTick( () => { |
||||||
|
gProgressPanel.addStatusBlock( "Uploading the PDF..." ) ; |
||||||
|
// read the selected file
|
||||||
|
let fileReader = new FileReader() ; |
||||||
|
fileReader.onload = () => { |
||||||
|
let pdfData = fileReader.result ; |
||||||
|
pdfData = removeBase64Prefix( pdfData ) ; |
||||||
|
this.uploadPdfData( pdfData ) ; |
||||||
|
} ; |
||||||
|
fileReader.readAsDataURL( file ) ; |
||||||
|
} ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
uploadPdfData( pdfData ) { |
||||||
|
// upload the PDF file to the backend
|
||||||
|
let data = { pdfData: pdfData } ; |
||||||
|
if ( gUrlParams.get( "test" ) ) { |
||||||
|
[ "npasses", "status", "warnings", "errors", "delay" ].forEach( (arg) => { |
||||||
|
let val = gUrlParams.get( arg ) ; |
||||||
|
if ( val ) |
||||||
|
data[arg] = val ; |
||||||
|
} ) ; |
||||||
|
} |
||||||
|
$.ajax( { |
||||||
|
url: gPrepareDataFilesUrl, //eslint-disable-line no-undef
|
||||||
|
type: "POST", |
||||||
|
data: JSON.stringify( data ), |
||||||
|
contentType: "application/json", |
||||||
|
} ).done( () => { |
||||||
|
// tell the backend to start processing
|
||||||
|
gProgressPanel.socketIOClient.emit( "start" ) ; |
||||||
|
} ).fail( (xhr, status, errorMsg) => { |
||||||
|
this.fatalErrorMsg = "Couldn't start processing: " + errorMsg ; |
||||||
|
} ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
onDone( downloadUrl ) { |
||||||
|
// make the download available to the user
|
||||||
|
$( this.$refs.progressPanel.$el ).css( { |
||||||
|
background: "#f0f0f0", color: "#444", |
||||||
|
"border-color": "#aaa", |
||||||
|
} ) ; |
||||||
|
this.downloadUrl = downloadUrl ; |
||||||
|
}, |
||||||
|
|
||||||
|
onFatalError( msg ) { |
||||||
|
this.fatalErrorMsg = msg ; |
||||||
|
}, |
||||||
|
|
||||||
|
}, |
||||||
|
|
||||||
|
} ) ; |
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
|
||||||
|
gPrepareApp.component( "upload-panel", { |
||||||
|
|
||||||
|
data() { return { |
||||||
|
isTestMode: gUrlParams.get( "test" ), |
||||||
|
uploadIconUrl: makeImageUrl( "eASLRB.png" ), |
||||||
|
} ; }, |
||||||
|
|
||||||
|
template: ` |
||||||
|
<div id="upload-panel"> |
||||||
|
<div v-if=isTestMode> |
||||||
|
<button @click=startTest style="height:28px;" > Go </button> |
||||||
|
Click on the button to start a test run. |
||||||
|
</div> |
||||||
|
<div v-else style="display:flex;"> |
||||||
|
<input type="file" @change=onFileSelected accept=".pdf" style="display:none;" ref="selectFile" > |
||||||
|
<button @click=onUploadProxy id="upload-proxy" ref="uploadProxy"> <img :src=uploadIconUrl /> </button> |
||||||
|
<div> Click on the button, and select your copy <br> of MMP's electronic ASLRB. |
||||||
|
<div class="info"> You <u>must</u> use the <a href="https://www.wargamevault.com/product/344879/Electronic-Advanced-Squad-Leader-Rulebook" target="_blank">offical MMP eASLRB</a>. <br> |
||||||
|
A scanned copy of a printed RB <u>will not work</u>! |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
</div>`, |
||||||
|
|
||||||
|
methods: { |
||||||
|
|
||||||
|
onUploadProxy() { |
||||||
|
// check if the test suite has left us some PDF file data to use
|
||||||
|
let $elem = $( "#testing-zip-data" ) ; |
||||||
|
if ( $elem.length > 0 && $elem.val().length > 0 ) { |
||||||
|
// yup - just return that
|
||||||
|
this.$emit( "file-selected", $elem.val() ) ; |
||||||
|
$elem.val( "" ) ; |
||||||
|
return ; |
||||||
|
} |
||||||
|
$elem.remove() ; // nb: this tells download-panel we are not being run by the test suite
|
||||||
|
// NOTE: It's difficult to style a file <input> element, so we make it hidden, and present
|
||||||
|
// a <button> element to the user, that clicks on the real file <input> when it is clicked.
|
||||||
|
this.$refs.selectFile.click() ; |
||||||
|
}, |
||||||
|
|
||||||
|
onFileSelected( evt ) { |
||||||
|
// NOTE: We would normally read the file here, but it takes some time because of its size,
|
||||||
|
// so we return the file object to the parent, so it can close us and open the progress panel,
|
||||||
|
// showing the "Uploading PDF" message, *then* we read the file and upload it.
|
||||||
|
this.$emit( "file-selected", evt.target.files[0] ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
startTest() { |
||||||
|
this.$emit( "file-selected", null ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
}, |
||||||
|
|
||||||
|
} ) ; |
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
|
||||||
|
gPrepareApp.component( "progress-panel", { |
||||||
|
|
||||||
|
data() { return { |
||||||
|
socketIOClient: null, |
||||||
|
statusBlocks: [], |
||||||
|
} ; }, |
||||||
|
|
||||||
|
template: ` |
||||||
|
<div id="progress-panel"> |
||||||
|
<status-block v-for="sb in statusBlocks" :statusBlock=sb :key=sb /> |
||||||
|
</div>`, |
||||||
|
|
||||||
|
created() { |
||||||
|
// initialize
|
||||||
|
gProgressPanel = this ; |
||||||
|
this.initSocketIOClient() ; |
||||||
|
}, |
||||||
|
|
||||||
|
methods: { |
||||||
|
|
||||||
|
initSocketIOClient() { |
||||||
|
// initialize the socketio client
|
||||||
|
let done = false ; |
||||||
|
this.socketIOClient = io.connect() ; //eslint-disable-line no-undef
|
||||||
|
this.socketIOClient.on( "disconnect", () => { |
||||||
|
if ( ! done ) |
||||||
|
this.$emit( "fatal", "The server has gone away. Please restart it, then reload this page." ) ; |
||||||
|
} ) ; |
||||||
|
this.socketIOClient.on( "status", (msg) => { this.addStatusBlock( msg ) ; } ) ; |
||||||
|
this.socketIOClient.on( "progress", (msg) => { this.addProgressMsg( "info", msg ) ; } ) ; |
||||||
|
this.socketIOClient.on( "warning", (msg) => { this.addProgressMsg( "warning", msg ) ; } ) ; |
||||||
|
this.socketIOClient.on( "error", (msg) => { this.addProgressMsg( "error", msg ) ; } ) ; |
||||||
|
this.socketIOClient.on( "done", (downloadUrl) => { |
||||||
|
done = true ; |
||||||
|
gProgressPanel.addStatusBlock( "All done." ) ; |
||||||
|
this.socketIOClient.disconnect() ; |
||||||
|
this.socketIOClient = null ; |
||||||
|
this.$emit( "done", downloadUrl ) ; |
||||||
|
} ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
addStatusBlock( statusMsg ) { |
||||||
|
// de-activate the previous status block
|
||||||
|
if ( this.statusBlocks.length > 0 ) |
||||||
|
this.statusBlocks[ this.statusBlocks.length-1 ].isActive = false ; |
||||||
|
// start a new status block
|
||||||
|
this.statusBlocks.push( { |
||||||
|
status: statusMsg, progress: [], |
||||||
|
isActive: true |
||||||
|
} ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
addProgressMsg( msgType, msg ) { |
||||||
|
// add a progress message to the current status block
|
||||||
|
if ( this.statusBlocks.length == 0 ) |
||||||
|
this.addStatusBlock( "" ) ; |
||||||
|
this.statusBlocks[ this.statusBlocks.length-1 ].progress.push( [ msgType, msg ] ) ; |
||||||
|
}, |
||||||
|
|
||||||
|
}, |
||||||
|
|
||||||
|
} ) ; |
||||||
|
|
||||||
|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||||
|
|
||||||
|
gPrepareApp.component( "status-block", { |
||||||
|
|
||||||
|
props: [ "statusBlock" ], |
||||||
|
|
||||||
|
template: ` |
||||||
|
<div class="status"> |
||||||
|
<div class="caption"> {{statusBlock.status}} </div> |
||||||
|
<table v-if="statusBlock.progress.length > 0" > |
||||||
|
<tr v-for="(p,pno) in statusBlock.progress" v-show="showProgress(p,pno)" > |
||||||
|
<td> <img :src=makeIconUrl(p) :style=makeIconCss(p) class="icon" /> </td> |
||||||
|
<td v-html=p[1] /> |
||||||
|
</tr> |
||||||
|
</table> |
||||||
|
</div>`, |
||||||
|
|
||||||
|
methods: { |
||||||
|
|
||||||
|
showProgress( progress, progressNo ) { |
||||||
|
// figure out if we should show a progress message or not
|
||||||
|
if ( progress[0] != "info" ) |
||||||
|
return true ; // nb: always show warnings/errors
|
||||||
|
if ( this.statusBlock.isActive && progressNo == this.statusBlock.progress.length-1 ) |
||||||
|
return true ; // nb: show the last progress message of the last status block
|
||||||
|
return false ; |
||||||
|
}, |
||||||
|
|
||||||
|
makeIconUrl( progress ) { |
||||||
|
if ( progress[0] == "info" ) |
||||||
|
return makeImageUrl( "bullet2.png" ) ; |
||||||
|
return makeImageUrl( progress[0] + ".png" ) ; |
||||||
|
}, |
||||||
|
makeIconCss( progress ) { |
||||||
|
if ( progress[0] == "info" ) |
||||||
|
return "height: 8px ; padding-left: 4px ;" ; |
||||||
|
}, |
||||||
|
|
||||||
|
}, |
||||||
|
|
||||||
|
} ) ; |
||||||
|
|
||||||
|
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||||
|
|
||||||
|
gPrepareApp.component( "download-panel", { |
||||||
|
|
||||||
|
props: [ "downloadUrl" ], |
||||||
|
data() { return { |
||||||
|
downloadIconUrl: makeImageUrl( "download.png" ), |
||||||
|
} ; }, |
||||||
|
|
||||||
|
template: ` |
||||||
|
<div id="download-panel"> |
||||||
|
<div style="display:flex;margin-bottom:10px;"> |
||||||
|
<button @click=onDownload id="download"> <img :src=downloadIconUrl /> </button> |
||||||
|
<div> Your data files are ready. |
||||||
|
<p> Click on the button to download them, and unpack them into a directory somewhere. </p> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
<div> Then restart the server with a <span class="pre">--data</span> parameter pointing to that directory e.g. |
||||||
|
<code> ./run-server.py --data ... </code> |
||||||
|
or |
||||||
|
<code> ./run-container.sh --data ... </code> |
||||||
|
</div> |
||||||
|
<div class="info"> |
||||||
|
You can edit these files directly, if you want to make changes. |
||||||
|
<p> If you want to make changes permanent (so they happen if you redo this preparation process), check out the files in <span class="pre">$/asl_rulebook2/extract//data/</span>. </p>
|
||||||
|
</div> |
||||||
|
</div>`, |
||||||
|
|
||||||
|
methods: { |
||||||
|
|
||||||
|
onDownload() { |
||||||
|
if ( ! this.downloadUrl ) { |
||||||
|
alert( "The download is not ready." ) ; // nb: should never get here!
|
||||||
|
return ; |
||||||
|
} |
||||||
|
// check if we are being run by the test suite
|
||||||
|
let $elem = $( "#testing-zip-data" ) ; |
||||||
|
if ( $elem.length == 0 ) { |
||||||
|
// nope - just return the download directly to the user
|
||||||
|
window.location.href = this.downloadUrl ; |
||||||
|
return ; |
||||||
|
} |
||||||
|
// yup - download the ZIP file and make it available to the test suite
|
||||||
|
// FUDGE! Setting the response type in a jQuery Ajax request:
|
||||||
|
// $.ajax( { type: "GET", url: ...,
|
||||||
|
// xhrFields: { responseType: "arraybuffer" }
|
||||||
|
// } ) ;
|
||||||
|
// should work, but doesn't :-/ Instead, we do it by providing a custom XHR object
|
||||||
|
// to manage the download. Things are slow, but this only used by the test suite.
|
||||||
|
let xhrOverride = new XMLHttpRequest() ; |
||||||
|
xhrOverride.responseType = "blob" ; |
||||||
|
$.ajax( { |
||||||
|
type: "GET", url: this.downloadUrl, |
||||||
|
xhr: function() { return xhrOverride ; }, |
||||||
|
} ).done( () => { |
||||||
|
// read the response
|
||||||
|
let fileReader = new FileReader() ; |
||||||
|
fileReader.onload = function( evt ){ |
||||||
|
let zip_data = evt.target.result ; |
||||||
|
// make the response available to the test suite
|
||||||
|
$( "#testing-zip-data" ).val( removeBase64Prefix( zip_data ) ) ; |
||||||
|
}; |
||||||
|
fileReader.readAsDataURL( xhrOverride.response ) ; |
||||||
|
} ).fail( (xhr, status, errorMsg) => { |
||||||
|
alert( "Download failed: " + errorMsg ) ; |
||||||
|
} ) ; |
||||||
|
}, |
||||||
|
}, |
||||||
|
|
||||||
|
} ) ; |
||||||
|
|
||||||
|
// --------------------------------------------------------------------
|
||||||
|
|
||||||
|
function makeImageUrl( fname ) { |
||||||
|
return gImagesBaseUrl + "/" + fname ; //eslint-disable-line no-undef
|
||||||
|
} |
||||||
|
|
||||||
|
function removeBase64Prefix( val ) |
||||||
|
{ |
||||||
|
// remove the base64 prefix from the start of the string
|
||||||
|
// - data: MIME-TYPE ; base64 , ...
|
||||||
|
return val.replace( /^data:.*?;base64,/, "" ) ; |
||||||
|
} |
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@ -0,0 +1,39 @@ |
|||||||
|
<!DOCTYPE html> |
||||||
|
<html lang="en"> |
||||||
|
|
||||||
|
<head> |
||||||
|
<meta charset="utf-8"> |
||||||
|
<title> Prepare {{APP_NAME}} data </title> |
||||||
|
<link rel="shortcut icon" href="{{url_for('static', filename='images/favicon.ico')}}"> |
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', |
||||||
|
filename = 'jquery-ui/jquery-ui' + WEB_DEBUG_MIN + '.css' |
||||||
|
) }}" /> |
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', filename='css/global.css' ) }}" /> |
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', filename='css/prepare.css' ) }}" /> |
||||||
|
</head> |
||||||
|
|
||||||
|
<body> |
||||||
|
<div id="prepare-app"></div> |
||||||
|
</body> |
||||||
|
|
||||||
|
{%if WEB_DEBUG%} |
||||||
|
<script src="{{ url_for( 'static', filename='vue/vue.global.js' ) }}"></script> |
||||||
|
{%else%} |
||||||
|
<script src="{{ url_for( 'static', filename='vue/vue.global.prod.js' ) }}"></script> |
||||||
|
{%endif%} |
||||||
|
<script src="{{ url_for( 'static', filename='jquery/jquery-3.6.0.js') }}"></script> |
||||||
|
<script src="{{ url_for( 'static', |
||||||
|
filename = 'jquery-ui/jquery-ui' + WEB_DEBUG_MIN + '.js' |
||||||
|
) }}"></script> |
||||||
|
<script src="{{ url_for( 'static', |
||||||
|
filename = 'socketio/socket.io' + WEB_DEBUG_MIN + '.js' |
||||||
|
) }}"></script> |
||||||
|
|
||||||
|
<script type="module" src="{{ url_for( 'static', filename='prepare.js' ) }}"></script> |
||||||
|
<script> |
||||||
|
gHaveGhostscript = "{{HAVE_GHOSTSCRIPT}}" ; |
||||||
|
gImagesBaseUrl = "{{ url_for( 'static', filename='images/' ) }}" ; |
||||||
|
gPrepareDataFilesUrl = "{{ url_for( 'prepare_data_files' ) }}" ; |
||||||
|
</script> |
||||||
|
|
||||||
|
</html> |
@ -0,0 +1,126 @@ |
|||||||
|
""" Test preparing the data files. """ |
||||||
|
|
||||||
|
import os |
||||||
|
import json |
||||||
|
import zipfile |
||||||
|
import io |
||||||
|
import base64 |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from asl_rulebook2.tests.utils import for_each_easlrb_version |
||||||
|
from asl_rulebook2.webapp.tests import pytest_options |
||||||
|
from asl_rulebook2.webapp.tests.utils import init_webapp, \ |
||||||
|
find_child, find_children, wait_for, wait_for_elem |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
@pytest.mark.skipif( not pytest_options.enable_prepare, reason="Prepare tests are not enabled." ) |
||||||
|
def test_prepare_logging( webapp, webdriver ): |
||||||
|
"""Test logging during the prepare process.""" |
||||||
|
|
||||||
|
# initialize |
||||||
|
# NOTE: We load the webapp without setting a data directory first. |
||||||
|
init_webapp( webapp, webdriver, |
||||||
|
test=1, npasses=50, warnings="25,27,42", errors="39,43", delay=0 |
||||||
|
) |
||||||
|
|
||||||
|
# generate some progress messages, check the results |
||||||
|
find_child( "#upload-panel button" ).click() |
||||||
|
def check_progress(): |
||||||
|
progress = _unload_progress() |
||||||
|
return progress == [ |
||||||
|
[ "Status #1", [] ], |
||||||
|
[ "Status #2", [] ], |
||||||
|
[ "Status #3", [ |
||||||
|
[ "warning.png", "Progress 25: warning" ], |
||||||
|
[ "warning.png", "Progress 27: warning" ], |
||||||
|
] ], |
||||||
|
[ "Status #4", [ |
||||||
|
[ "error.png", "Progress 39: error" ] |
||||||
|
] ], |
||||||
|
[ "Status #5", [ |
||||||
|
[ "warning.png", "Progress 42: warning" ], |
||||||
|
[ "error.png", "Progress 43: error" ], |
||||||
|
] ], |
||||||
|
[ "All done.", [] ] |
||||||
|
] |
||||||
|
wait_for( 2, check_progress ) |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
@pytest.mark.skipif( not pytest_options.enable_prepare, reason="Prepare tests are not enabled." ) |
||||||
|
@pytest.mark.skipif( not pytest_options.easlrb_path, reason="eASLRB not available." ) |
||||||
|
def test_full_prepare( webapp, webdriver ): |
||||||
|
"""Test the full prepare process.""" |
||||||
|
|
||||||
|
def do_test( dname ): |
||||||
|
|
||||||
|
# initialize |
||||||
|
# NOTE: We load the webapp without setting a data directory first. |
||||||
|
init_webapp( webapp, webdriver ) |
||||||
|
|
||||||
|
# load the PDF file data into the web page (since we can't manipulate the "Open File" dialog) |
||||||
|
fname = os.path.join( dname, "eASLRB.pdf" ) |
||||||
|
with open( fname, "rb" ) as fp: |
||||||
|
zip_data = fp.read() |
||||||
|
testing_zip_data = find_child( "#testing-zip-data", webdriver ) |
||||||
|
webdriver.execute_script( "arguments[0].value = arguments[1]", testing_zip_data, |
||||||
|
base64.b64encode( zip_data ).decode( "ascii" ) |
||||||
|
) |
||||||
|
|
||||||
|
# start the prepare process, and wait for it to finish |
||||||
|
# NOTE: It will have auto-started because we passed in a filename to the webapp. |
||||||
|
find_child( "button#upload-proxy" ).click() |
||||||
|
wait_for_elem( 30*60, "#download-panel" ) |
||||||
|
|
||||||
|
# get the results |
||||||
|
find_child( "button#download" ).click() |
||||||
|
zip_data = wait_for( 20, lambda: testing_zip_data.get_attribute( "value" ) ) |
||||||
|
zip_data = base64.b64decode( zip_data ) |
||||||
|
|
||||||
|
# check the results |
||||||
|
with zipfile.ZipFile( io.BytesIO( zip_data ) ) as zip_file: |
||||||
|
assert set( zip_file.namelist() ) == set( [ |
||||||
|
"ASL Rulebook.pdf", "ASL Rulebook.index", |
||||||
|
"ASL Rulebook.targets", "ASL Rulebook.chapters", "ASL Rulebook.footnotes" |
||||||
|
] ) |
||||||
|
assert zip_file.getinfo( "ASL Rulebook.pdf" ).file_size > 40*1000 |
||||||
|
for ftype in [ "index", "targets", "chapters", "footnotes" ]: |
||||||
|
fname = os.path.join( dname, ftype+".json" ) |
||||||
|
expected = json.load( open( fname, "r" ) ) |
||||||
|
fdata = zip_file.read( "ASL Rulebook.{}".format( ftype ) ) |
||||||
|
assert json.loads( fdata ) == expected |
||||||
|
|
||||||
|
# run the test |
||||||
|
for_each_easlrb_version( do_test ) |
||||||
|
|
||||||
|
# --------------------------------------------------------------------- |
||||||
|
|
||||||
|
def _unload_progress(): |
||||||
|
"""Unload the progress messages.""" |
||||||
|
|
||||||
|
def unload_status_block( root ): |
||||||
|
"""Unload a status block and its progress messages.""" |
||||||
|
caption = find_child( ".caption", root ).text |
||||||
|
msgs = [ |
||||||
|
unload_msg( row ) |
||||||
|
for row in find_children( "tr", root ) |
||||||
|
if row.is_displayed() |
||||||
|
] |
||||||
|
return [ caption, msgs ] |
||||||
|
|
||||||
|
def unload_msg( row ): |
||||||
|
"""Unload a single progress message.""" |
||||||
|
cells = find_children( "td", row ) |
||||||
|
assert len(cells) == 2 |
||||||
|
img = find_child( "img", cells[0] ) |
||||||
|
url = img.get_attribute( "src" ) |
||||||
|
return [ os.path.basename(url), cells[1].text ] |
||||||
|
|
||||||
|
# unload each status block |
||||||
|
progress_panel = find_child( "#progress-panel" ) |
||||||
|
return [ |
||||||
|
unload_status_block( elem ) |
||||||
|
for elem in find_children( ".status", progress_panel ) |
||||||
|
] |
Loading…
Reference in new issue