parent
fdd027cb2b
commit
fee9f49f1c
@ -0,0 +1,19 @@ |
||||
""" Helper utilities. """ |
||||
|
||||
import os |
||||
|
||||
from asl_rulebook2.tests import pytest_options |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def for_each_easlrb_version( func ): |
||||
"""Run tests for each version of the eASLRB.""" |
||||
assert pytest_options.easlrb_path |
||||
base_dir = pytest_options.easlrb_path |
||||
ncalls = 0 |
||||
for name in os.listdir( base_dir ): |
||||
dname = os.path.join( base_dir, name ) |
||||
if os.path.isfile( os.path.join( dname, "eASLRB.pdf" ) ): |
||||
func( dname ) |
||||
ncalls += 1 |
||||
assert ncalls > 0 |
||||
@ -0,0 +1,214 @@ |
||||
""" Analyze the MMP eASLRB PDF and prepare the data files. """ |
||||
|
||||
import threading |
||||
import zipfile |
||||
import io |
||||
import time |
||||
import base64 |
||||
import traceback |
||||
import logging |
||||
|
||||
from flask import request, send_file, abort, url_for |
||||
|
||||
from asl_rulebook2.extract.all import ExtractAll |
||||
from asl_rulebook2.bin.prepare_pdf import prepare_pdf |
||||
from asl_rulebook2.bin.fixup_mmp_pdf import fixup_mmp_pdf |
||||
from asl_rulebook2.pdf import PdfDoc |
||||
from asl_rulebook2.utils import TempFile |
||||
from asl_rulebook2.webapp import app, globvars |
||||
from asl_rulebook2.webapp.utils import get_gs_path |
||||
|
||||
_zip_data_download = None |
||||
|
||||
_logger = logging.getLogger( "prepare" ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/prepare", methods=["POST"] ) |
||||
def prepare_data_files(): |
||||
"""Prepare the data files.""" |
||||
|
||||
# initialize |
||||
args = dict( request.json ) |
||||
download_url = url_for( "download_prepared_data" ) |
||||
|
||||
# initialize the socketio server |
||||
sio = globvars.socketio_server |
||||
if not sio: |
||||
raise RuntimeError( "The socketio server has not been started." ) |
||||
@sio.on( "start" ) |
||||
def on_start( data ): #pylint: disable=unused-variable,unused-argument |
||||
# start the worker thread that prepares the data files |
||||
# NOTE: We don't do this when the POST request comes in, but wait until the client |
||||
# tells us it's ready (otherwise, it might miss the first event or two). |
||||
def worker(): |
||||
try: |
||||
_do_prepare_data_files( args, download_url ) |
||||
except Exception as ex: #pylint: disable=broad-except |
||||
_logger.error( "PREPARE ERROR: %s\n%s", ex, traceback.format_exc() ) |
||||
globvars.socketio_server.emit( "error", str(ex) ) |
||||
threading.Thread( target=worker, daemon=True ).start() |
||||
|
||||
return "ok" |
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
||||
|
||||
def _do_prepare_data_files( args, download_url ): |
||||
|
||||
# initialize |
||||
sio = globvars.socketio_server |
||||
pdf_data = args.get( "pdfData" ) |
||||
if not pdf_data: |
||||
# no data was sent - this is a test of logging progress messages. |
||||
del args["pdfData"] |
||||
_test_progress( **args ) |
||||
return |
||||
pdf_data = base64.b64decode( pdf_data ) |
||||
|
||||
def on_done( zip_data ): |
||||
global _zip_data_download |
||||
_zip_data_download = zip_data |
||||
sio.emit( "done", download_url ) |
||||
|
||||
# check if we should just return a pre-prepared ZIP file (for testing porpoises) |
||||
fname = app.config.get( "PREPARED_ZIP" ) |
||||
if fname: |
||||
with open( fname, "rb" ) as fp: |
||||
on_done( fp.read() ) |
||||
return |
||||
|
||||
with TempFile() as input_file, TempFile() as prepared_file: |
||||
|
||||
# save the PDF file data |
||||
input_file.write( pdf_data ) |
||||
input_file.close( delete=False ) |
||||
_logger.info( "Saved PDF file (#bytes=%d): %s", len(pdf_data), input_file.name ) |
||||
|
||||
# initialize logging |
||||
msg_types = set() |
||||
def log_msg( msg_type, msg ): |
||||
msg = msg.lstrip() |
||||
if msg_type == "status": |
||||
_logger.info( "[STATUS]: %s", msg ) |
||||
elif msg_type == "warning": |
||||
_logger.warning( "[WARNING]: %s", msg ) |
||||
elif msg_type == "error": |
||||
_logger.error( "[ERROR]: %s", msg ) |
||||
else: |
||||
_logger.debug( "[%s] %s", msg_type, msg ) |
||||
if msg.startswith( "- " ): |
||||
msg = msg[2:] |
||||
sio.emit( msg_type, msg ) |
||||
msg_types.add( msg_type ) |
||||
|
||||
# NOTE: The plan was to allow the user to change the default parameters in the UI, |
||||
# but this can be done (ahem) later. For now, if they really need to change something, |
||||
# they can prepare the data files from the command-line. |
||||
args = [] |
||||
|
||||
# extract everything we need from the PDF |
||||
log_msg( "status", "Opening the PDF..." ) |
||||
extract = ExtractAll( args, log_msg ) |
||||
with PdfDoc( input_file.name ) as pdf: |
||||
extract.extract_all( pdf ) |
||||
index_buf = io.StringIO() |
||||
extract.extract_index.save_as_json( index_buf ) |
||||
targets_buf, chapters_buf, footnotes_buf = io.StringIO(), io.StringIO(), io.StringIO() |
||||
extract.extract_content.save_as_json( targets_buf, chapters_buf, footnotes_buf ) |
||||
file_data = { |
||||
"index": index_buf.getvalue(), |
||||
"targets": targets_buf.getvalue(), |
||||
"chapters": chapters_buf.getvalue(), |
||||
"footnotes": footnotes_buf.getvalue(), |
||||
} |
||||
|
||||
# prepare the PDF |
||||
gs_path = get_gs_path() |
||||
if not gs_path: |
||||
raise RuntimeError( "Ghostscript is not available." ) |
||||
with TempFile( mode="w", encoding="utf-8" ) as targets_file: |
||||
log_msg( "status", "Preparing the final PDF..." ) |
||||
# save the extracted targets |
||||
targets_file.temp_file.write( file_data["targets"] ) |
||||
targets_file.close( delete=False ) |
||||
# prepare the PDF |
||||
prepared_file.close( delete=False ) |
||||
prepare_pdf( input_file.name, |
||||
"ASL Rulebook", |
||||
targets_file.name, 5, |
||||
prepared_file.name, "ebook", |
||||
gs_path, |
||||
log_msg |
||||
) |
||||
|
||||
# fixup the PDF |
||||
with TempFile() as fixedup_file: |
||||
log_msg( "status", "Fixing up the final PDF..." ) |
||||
fixedup_file.close( delete=False ) |
||||
fixup_mmp_pdf( prepared_file.name, |
||||
fixedup_file.name, |
||||
True, True, |
||||
log_msg |
||||
) |
||||
# read the final PDF data |
||||
with open( fixedup_file.name, "rb" ) as fp: |
||||
pdf_data = fp.read() |
||||
|
||||
# prepare the ZIP for the user to download |
||||
log_msg( "status", "Preparing the download ZIP..." ) |
||||
zip_data = io.BytesIO() |
||||
with zipfile.ZipFile( zip_data, "w", zipfile.ZIP_DEFLATED ) as zip_file: |
||||
fname_stem = "ASL Rulebook" |
||||
zip_file.writestr( fname_stem+".pdf", pdf_data ) |
||||
for key in file_data: |
||||
fname = "{}.{}".format( fname_stem, key ) |
||||
zip_file.writestr( fname, file_data[key] ) |
||||
zip_data = zip_data.getvalue() |
||||
|
||||
# notify the front-end that we're done |
||||
on_done( zip_data ) |
||||
_logger.debug( "Message types seen: %s", |
||||
" ; ".join( sorted( str(mt) for mt in msg_types ) ) |
||||
) |
||||
|
||||
# NOTE: We don't bother shutting down the socketio server, since the user |
||||
# has to restart the server, using the newly-prepared data files. |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/prepare/download" ) |
||||
def download_prepared_data(): |
||||
"""Download the prepared data ZIP file.""" |
||||
if not _zip_data_download: |
||||
abort( 404 ) |
||||
return send_file( |
||||
io.BytesIO( _zip_data_download ), |
||||
as_attachment=True, attachment_filename="asl-rulebook2.zip" |
||||
) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _test_progress( npasses=100, status=10, warnings=None, errors=None, delay=0.1 ): |
||||
"""Test progress messages.""" |
||||
|
||||
# initialize |
||||
warnings = [ int(w) for w in warnings.split(",") ] if warnings else [] |
||||
errors = [ int(e) for e in errors.split(",") ] if errors else [] |
||||
|
||||
# generate progress messages |
||||
sio = globvars.socketio_server |
||||
status_no = 0 |
||||
for i in range( int(npasses) ): |
||||
# check if we should start a new status block |
||||
if i % status == 0: |
||||
status_no += 1 |
||||
sio.emit( "status", "Status #{}".format( status_no ) ) |
||||
# issue the next progress message |
||||
if 1+i in warnings: |
||||
sio.emit( "warning", "Progress {}: warning".format( 1+i ) ) |
||||
if 1+i in errors: |
||||
sio.emit( "error", "Progress {}: error".format( 1+i ) ) |
||||
else: |
||||
sio.emit( "progress", "Progress {}.".format( 1+i ) ) |
||||
time.sleep( float( delay ) ) |
||||
sio.emit( "done" ) |
||||
@ -0,0 +1,37 @@ |
||||
p { margin: 5px 0 ; } |
||||
code { display: block ; margin: 5px 0 5px 20px ; } |
||||
.info { |
||||
margin-top: 10px ; min-height: 25px ; |
||||
padding-left: 30px ; background: no-repeat url(../images/info.png) ; |
||||
font-size: 80% ; font-style: italic ; color: #444 ; |
||||
} |
||||
|
||||
#prepare-app { height: 100% ; display: flex ; } |
||||
#header { margin-bottom: 5px ; } |
||||
#main { width: 100% ; margin: 10px ; display: flex ; flex-direction: column ; } |
||||
|
||||
#fatal-error { margin-bottom: 10px ; font-size: 120% ; font-weight: bold ; } |
||||
|
||||
#upload-panel { align-self: start ; border: 1px solid black ; border-radius: 5px ; padding: 10px ; } |
||||
#upload-panel button { height: 70px ; margin-right: 10px ; } |
||||
#upload-panel button img { margin-top: 3px ; height: 60px ; } |
||||
|
||||
#progress-panel { |
||||
flex-grow: 1 ; overflow-y: auto ; |
||||
border: 1px solid black ; border-radius: 5px ; padding: 10px ; |
||||
font-family: monospace ; font-size: 90% ; |
||||
} |
||||
#progress-panel .progress { font-style: italic ; } |
||||
#progress-panel .status { margin: 5px 0 ; } |
||||
#progress-panel .status:first-of-type { margin-top: 0 ; } |
||||
#progress-panel .status table { margin-left: 2px ; } |
||||
#progress-panel .status table td { vertical-align: top ; } |
||||
#progress-panel .status img.icon { height: 15px ; margin: 1px 3px 0 0 ; } |
||||
|
||||
#download-panel { |
||||
position: fixed ; bottom: 18px ; right: 18px ; width: 75% ; |
||||
border: 1px solid black ; border-radius: 5px ; background: white ; |
||||
padding: 10px ; |
||||
} |
||||
#download-panel button { height: 40px ; margin-right: 10px ; padding: 5px ; } |
||||
#download-panel button img { height: 30px ; } |
||||
|
After Width: | Height: | Size: 2.5 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
@ -0,0 +1,364 @@ |
||||
// create the main application
|
||||
export const gPrepareApp = Vue.createApp( { //eslint-disable-line no-undef
|
||||
template: "<prepare-app />", |
||||
} ) ; |
||||
$(document).ready( () => { |
||||
gPrepareApp.mount( "#prepare-app" ) ; |
||||
} ) ; |
||||
|
||||
// parse any URL parameters
|
||||
let gUrlParams = new URLSearchParams( window.location.search.substring(1) ) ; |
||||
|
||||
let gProgressPanel = null ; |
||||
|
||||
// --------------------------------------------------------------------
|
||||
|
||||
gPrepareApp.component( "prepare-app", { |
||||
|
||||
data() { return { |
||||
isLoaded: false, |
||||
isProcessing: false, |
||||
downloadUrl: null, |
||||
fatalErrorMsg: gHaveGhostscript ? null : "Ghostscript is not available.", //eslint-disable-line no-undef
|
||||
fatalErrorIconUrl: makeImageUrl( "error.png" ), |
||||
} ; }, |
||||
|
||||
template: ` |
||||
<div id="main"> |
||||
<div id="header"> |
||||
No data directory has been configured. |
||||
<p> If you haven't used this program before, a few things need to be prepared first. |
||||
It will take around 10-15 minutes. </p> |
||||
</div> |
||||
<div v-show=fatalErrorMsg id="fatal-error" > |
||||
<img :src=fatalErrorIconUrl style="float:left;margin-right:5px;" /> |
||||
{{fatalErrorMsg}} |
||||
</div> |
||||
<upload-panel v-show="!fatalErrorMsg &&!isProcessing" @file-selected=onFileSelected /> |
||||
<progress-panel v-show=isProcessing @done=onDone @fatal=onFatalError ref=progressPanel /> |
||||
<download-panel v-show=downloadUrl :downloadUrl=downloadUrl ref=downloadPanel /> |
||||
<textarea id="testing-zip-data" style="display:none;" /> |
||||
<div v-if=isLoaded id="_prepareapp-loaded_" /> |
||||
</div>`, |
||||
|
||||
mounted() { |
||||
// initialize the UI
|
||||
$( "button" ).button() ; |
||||
this.isLoaded = true ; |
||||
}, |
||||
|
||||
methods: { |
||||
|
||||
onFileSelected( file ) { |
||||
this.isProcessing = true ; |
||||
if ( ! file ) { |
||||
// this is a test of progress logging
|
||||
this.uploadPdfData( null ) ; |
||||
return ; |
||||
} |
||||
if ( typeof file == "string" ) { |
||||
// this is PDF file data given to us by the test suite - just return it as is
|
||||
this.uploadPdfData( file ) ; |
||||
return ; |
||||
} |
||||
this.$nextTick( () => { |
||||
gProgressPanel.addStatusBlock( "Uploading the PDF..." ) ; |
||||
// read the selected file
|
||||
let fileReader = new FileReader() ; |
||||
fileReader.onload = () => { |
||||
let pdfData = fileReader.result ; |
||||
pdfData = removeBase64Prefix( pdfData ) ; |
||||
this.uploadPdfData( pdfData ) ; |
||||
} ; |
||||
fileReader.readAsDataURL( file ) ; |
||||
} ) ; |
||||
}, |
||||
|
||||
uploadPdfData( pdfData ) { |
||||
// upload the PDF file to the backend
|
||||
let data = { pdfData: pdfData } ; |
||||
if ( gUrlParams.get( "test" ) ) { |
||||
[ "npasses", "status", "warnings", "errors", "delay" ].forEach( (arg) => { |
||||
let val = gUrlParams.get( arg ) ; |
||||
if ( val ) |
||||
data[arg] = val ; |
||||
} ) ; |
||||
} |
||||
$.ajax( { |
||||
url: gPrepareDataFilesUrl, //eslint-disable-line no-undef
|
||||
type: "POST", |
||||
data: JSON.stringify( data ), |
||||
contentType: "application/json", |
||||
} ).done( () => { |
||||
// tell the backend to start processing
|
||||
gProgressPanel.socketIOClient.emit( "start" ) ; |
||||
} ).fail( (xhr, status, errorMsg) => { |
||||
this.fatalErrorMsg = "Couldn't start processing: " + errorMsg ; |
||||
} ) ; |
||||
}, |
||||
|
||||
onDone( downloadUrl ) { |
||||
// make the download available to the user
|
||||
$( this.$refs.progressPanel.$el ).css( { |
||||
background: "#f0f0f0", color: "#444", |
||||
"border-color": "#aaa", |
||||
} ) ; |
||||
this.downloadUrl = downloadUrl ; |
||||
}, |
||||
|
||||
onFatalError( msg ) { |
||||
this.fatalErrorMsg = msg ; |
||||
}, |
||||
|
||||
}, |
||||
|
||||
} ) ; |
||||
|
||||
// --------------------------------------------------------------------
|
||||
|
||||
gPrepareApp.component( "upload-panel", { |
||||
|
||||
data() { return { |
||||
isTestMode: gUrlParams.get( "test" ), |
||||
uploadIconUrl: makeImageUrl( "eASLRB.png" ), |
||||
} ; }, |
||||
|
||||
template: ` |
||||
<div id="upload-panel"> |
||||
<div v-if=isTestMode> |
||||
<button @click=startTest style="height:28px;" > Go </button> |
||||
Click on the button to start a test run. |
||||
</div> |
||||
<div v-else style="display:flex;"> |
||||
<input type="file" @change=onFileSelected accept=".pdf" style="display:none;" ref="selectFile" > |
||||
<button @click=onUploadProxy id="upload-proxy" ref="uploadProxy"> <img :src=uploadIconUrl /> </button> |
||||
<div> Click on the button, and select your copy <br> of MMP's electronic ASLRB. |
||||
<div class="info"> You <u>must</u> use the <a href="https://www.wargamevault.com/product/344879/Electronic-Advanced-Squad-Leader-Rulebook" target="_blank">offical MMP eASLRB</a>. <br> |
||||
A scanned copy of a printed RB <u>will not work</u>! |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</div>`, |
||||
|
||||
methods: { |
||||
|
||||
onUploadProxy() { |
||||
// check if the test suite has left us some PDF file data to use
|
||||
let $elem = $( "#testing-zip-data" ) ; |
||||
if ( $elem.length > 0 && $elem.val().length > 0 ) { |
||||
// yup - just return that
|
||||
this.$emit( "file-selected", $elem.val() ) ; |
||||
$elem.val( "" ) ; |
||||
return ; |
||||
} |
||||
$elem.remove() ; // nb: this tells download-panel we are not being run by the test suite
|
||||
// NOTE: It's difficult to style a file <input> element, so we make it hidden, and present
|
||||
// a <button> element to the user, that clicks on the real file <input> when it is clicked.
|
||||
this.$refs.selectFile.click() ; |
||||
}, |
||||
|
||||
onFileSelected( evt ) { |
||||
// NOTE: We would normally read the file here, but it takes some time because of its size,
|
||||
// so we return the file object to the parent, so it can close us and open the progress panel,
|
||||
// showing the "Uploading PDF" message, *then* we read the file and upload it.
|
||||
this.$emit( "file-selected", evt.target.files[0] ) ; |
||||
}, |
||||
|
||||
startTest() { |
||||
this.$emit( "file-selected", null ) ; |
||||
}, |
||||
|
||||
}, |
||||
|
||||
} ) ; |
||||
|
||||
// --------------------------------------------------------------------
|
||||
|
||||
gPrepareApp.component( "progress-panel", { |
||||
|
||||
data() { return { |
||||
socketIOClient: null, |
||||
statusBlocks: [], |
||||
} ; }, |
||||
|
||||
template: ` |
||||
<div id="progress-panel"> |
||||
<status-block v-for="sb in statusBlocks" :statusBlock=sb :key=sb /> |
||||
</div>`, |
||||
|
||||
created() { |
||||
// initialize
|
||||
gProgressPanel = this ; |
||||
this.initSocketIOClient() ; |
||||
}, |
||||
|
||||
methods: { |
||||
|
||||
initSocketIOClient() { |
||||
// initialize the socketio client
|
||||
let done = false ; |
||||
this.socketIOClient = io.connect() ; //eslint-disable-line no-undef
|
||||
this.socketIOClient.on( "disconnect", () => { |
||||
if ( ! done ) |
||||
this.$emit( "fatal", "The server has gone away. Please restart it, then reload this page." ) ; |
||||
} ) ; |
||||
this.socketIOClient.on( "status", (msg) => { this.addStatusBlock( msg ) ; } ) ; |
||||
this.socketIOClient.on( "progress", (msg) => { this.addProgressMsg( "info", msg ) ; } ) ; |
||||
this.socketIOClient.on( "warning", (msg) => { this.addProgressMsg( "warning", msg ) ; } ) ; |
||||
this.socketIOClient.on( "error", (msg) => { this.addProgressMsg( "error", msg ) ; } ) ; |
||||
this.socketIOClient.on( "done", (downloadUrl) => { |
||||
done = true ; |
||||
gProgressPanel.addStatusBlock( "All done." ) ; |
||||
this.socketIOClient.disconnect() ; |
||||
this.socketIOClient = null ; |
||||
this.$emit( "done", downloadUrl ) ; |
||||
} ) ; |
||||
}, |
||||
|
||||
addStatusBlock( statusMsg ) { |
||||
// de-activate the previous status block
|
||||
if ( this.statusBlocks.length > 0 ) |
||||
this.statusBlocks[ this.statusBlocks.length-1 ].isActive = false ; |
||||
// start a new status block
|
||||
this.statusBlocks.push( { |
||||
status: statusMsg, progress: [], |
||||
isActive: true |
||||
} ) ; |
||||
}, |
||||
|
||||
addProgressMsg( msgType, msg ) { |
||||
// add a progress message to the current status block
|
||||
if ( this.statusBlocks.length == 0 ) |
||||
this.addStatusBlock( "" ) ; |
||||
this.statusBlocks[ this.statusBlocks.length-1 ].progress.push( [ msgType, msg ] ) ; |
||||
}, |
||||
|
||||
}, |
||||
|
||||
} ) ; |
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
gPrepareApp.component( "status-block", { |
||||
|
||||
props: [ "statusBlock" ], |
||||
|
||||
template: ` |
||||
<div class="status"> |
||||
<div class="caption"> {{statusBlock.status}} </div> |
||||
<table v-if="statusBlock.progress.length > 0" > |
||||
<tr v-for="(p,pno) in statusBlock.progress" v-show="showProgress(p,pno)" > |
||||
<td> <img :src=makeIconUrl(p) :style=makeIconCss(p) class="icon" /> </td> |
||||
<td v-html=p[1] /> |
||||
</tr> |
||||
</table> |
||||
</div>`, |
||||
|
||||
methods: { |
||||
|
||||
showProgress( progress, progressNo ) { |
||||
// figure out if we should show a progress message or not
|
||||
if ( progress[0] != "info" ) |
||||
return true ; // nb: always show warnings/errors
|
||||
if ( this.statusBlock.isActive && progressNo == this.statusBlock.progress.length-1 ) |
||||
return true ; // nb: show the last progress message of the last status block
|
||||
return false ; |
||||
}, |
||||
|
||||
makeIconUrl( progress ) { |
||||
if ( progress[0] == "info" ) |
||||
return makeImageUrl( "bullet2.png" ) ; |
||||
return makeImageUrl( progress[0] + ".png" ) ; |
||||
}, |
||||
makeIconCss( progress ) { |
||||
if ( progress[0] == "info" ) |
||||
return "height: 8px ; padding-left: 4px ;" ; |
||||
}, |
||||
|
||||
}, |
||||
|
||||
} ) ; |
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
gPrepareApp.component( "download-panel", { |
||||
|
||||
props: [ "downloadUrl" ], |
||||
data() { return { |
||||
downloadIconUrl: makeImageUrl( "download.png" ), |
||||
} ; }, |
||||
|
||||
template: ` |
||||
<div id="download-panel"> |
||||
<div style="display:flex;margin-bottom:10px;"> |
||||
<button @click=onDownload id="download"> <img :src=downloadIconUrl /> </button> |
||||
<div> Your data files are ready. |
||||
<p> Click on the button to download them, and unpack them into a directory somewhere. </p> |
||||
</div> |
||||
</div> |
||||
<div> Then restart the server with a <span class="pre">--data</span> parameter pointing to that directory e.g. |
||||
<code> ./run-server.py --data ... </code> |
||||
or |
||||
<code> ./run-container.sh --data ... </code> |
||||
</div> |
||||
<div class="info"> |
||||
You can edit these files directly, if you want to make changes. |
||||
<p> If you want to make changes permanent (so they happen if you redo this preparation process), check out the files in <span class="pre">$/asl_rulebook2/extract//data/</span>. </p>
|
||||
</div> |
||||
</div>`, |
||||
|
||||
methods: { |
||||
|
||||
onDownload() { |
||||
if ( ! this.downloadUrl ) { |
||||
alert( "The download is not ready." ) ; // nb: should never get here!
|
||||
return ; |
||||
} |
||||
// check if we are being run by the test suite
|
||||
let $elem = $( "#testing-zip-data" ) ; |
||||
if ( $elem.length == 0 ) { |
||||
// nope - just return the download directly to the user
|
||||
window.location.href = this.downloadUrl ; |
||||
return ; |
||||
} |
||||
// yup - download the ZIP file and make it available to the test suite
|
||||
// FUDGE! Setting the response type in a jQuery Ajax request:
|
||||
// $.ajax( { type: "GET", url: ...,
|
||||
// xhrFields: { responseType: "arraybuffer" }
|
||||
// } ) ;
|
||||
// should work, but doesn't :-/ Instead, we do it by providing a custom XHR object
|
||||
// to manage the download. Things are slow, but this only used by the test suite.
|
||||
let xhrOverride = new XMLHttpRequest() ; |
||||
xhrOverride.responseType = "blob" ; |
||||
$.ajax( { |
||||
type: "GET", url: this.downloadUrl, |
||||
xhr: function() { return xhrOverride ; }, |
||||
} ).done( () => { |
||||
// read the response
|
||||
let fileReader = new FileReader() ; |
||||
fileReader.onload = function( evt ){ |
||||
let zip_data = evt.target.result ; |
||||
// make the response available to the test suite
|
||||
$( "#testing-zip-data" ).val( removeBase64Prefix( zip_data ) ) ; |
||||
}; |
||||
fileReader.readAsDataURL( xhrOverride.response ) ; |
||||
} ).fail( (xhr, status, errorMsg) => { |
||||
alert( "Download failed: " + errorMsg ) ; |
||||
} ) ; |
||||
}, |
||||
}, |
||||
|
||||
} ) ; |
||||
|
||||
// --------------------------------------------------------------------
|
||||
|
||||
function makeImageUrl( fname ) { |
||||
return gImagesBaseUrl + "/" + fname ; //eslint-disable-line no-undef
|
||||
} |
||||
|
||||
function removeBase64Prefix( val ) |
||||
{ |
||||
// remove the base64 prefix from the start of the string
|
||||
// - data: MIME-TYPE ; base64 , ...
|
||||
return val.replace( /^data:.*?;base64,/, "" ) ; |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@ -0,0 +1,39 @@ |
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
|
||||
<head> |
||||
<meta charset="utf-8"> |
||||
<title> Prepare {{APP_NAME}} data </title> |
||||
<link rel="shortcut icon" href="{{url_for('static', filename='images/favicon.ico')}}"> |
||||
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', |
||||
filename = 'jquery-ui/jquery-ui' + WEB_DEBUG_MIN + '.css' |
||||
) }}" /> |
||||
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', filename='css/global.css' ) }}" /> |
||||
<link rel="stylesheet" type="text/css" href="{{ url_for( 'static', filename='css/prepare.css' ) }}" /> |
||||
</head> |
||||
|
||||
<body> |
||||
<div id="prepare-app"></div> |
||||
</body> |
||||
|
||||
{%if WEB_DEBUG%} |
||||
<script src="{{ url_for( 'static', filename='vue/vue.global.js' ) }}"></script> |
||||
{%else%} |
||||
<script src="{{ url_for( 'static', filename='vue/vue.global.prod.js' ) }}"></script> |
||||
{%endif%} |
||||
<script src="{{ url_for( 'static', filename='jquery/jquery-3.6.0.js') }}"></script> |
||||
<script src="{{ url_for( 'static', |
||||
filename = 'jquery-ui/jquery-ui' + WEB_DEBUG_MIN + '.js' |
||||
) }}"></script> |
||||
<script src="{{ url_for( 'static', |
||||
filename = 'socketio/socket.io' + WEB_DEBUG_MIN + '.js' |
||||
) }}"></script> |
||||
|
||||
<script type="module" src="{{ url_for( 'static', filename='prepare.js' ) }}"></script> |
||||
<script> |
||||
gHaveGhostscript = "{{HAVE_GHOSTSCRIPT}}" ; |
||||
gImagesBaseUrl = "{{ url_for( 'static', filename='images/' ) }}" ; |
||||
gPrepareDataFilesUrl = "{{ url_for( 'prepare_data_files' ) }}" ; |
||||
</script> |
||||
|
||||
</html> |
||||
@ -0,0 +1,126 @@ |
||||
""" Test preparing the data files. """ |
||||
|
||||
import os |
||||
import json |
||||
import zipfile |
||||
import io |
||||
import base64 |
||||
|
||||
import pytest |
||||
|
||||
from asl_rulebook2.tests.utils import for_each_easlrb_version |
||||
from asl_rulebook2.webapp.tests import pytest_options |
||||
from asl_rulebook2.webapp.tests.utils import init_webapp, \ |
||||
find_child, find_children, wait_for, wait_for_elem |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@pytest.mark.skipif( not pytest_options.enable_prepare, reason="Prepare tests are not enabled." ) |
||||
def test_prepare_logging( webapp, webdriver ): |
||||
"""Test logging during the prepare process.""" |
||||
|
||||
# initialize |
||||
# NOTE: We load the webapp without setting a data directory first. |
||||
init_webapp( webapp, webdriver, |
||||
test=1, npasses=50, warnings="25,27,42", errors="39,43", delay=0 |
||||
) |
||||
|
||||
# generate some progress messages, check the results |
||||
find_child( "#upload-panel button" ).click() |
||||
def check_progress(): |
||||
progress = _unload_progress() |
||||
return progress == [ |
||||
[ "Status #1", [] ], |
||||
[ "Status #2", [] ], |
||||
[ "Status #3", [ |
||||
[ "warning.png", "Progress 25: warning" ], |
||||
[ "warning.png", "Progress 27: warning" ], |
||||
] ], |
||||
[ "Status #4", [ |
||||
[ "error.png", "Progress 39: error" ] |
||||
] ], |
||||
[ "Status #5", [ |
||||
[ "warning.png", "Progress 42: warning" ], |
||||
[ "error.png", "Progress 43: error" ], |
||||
] ], |
||||
[ "All done.", [] ] |
||||
] |
||||
wait_for( 2, check_progress ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@pytest.mark.skipif( not pytest_options.enable_prepare, reason="Prepare tests are not enabled." ) |
||||
@pytest.mark.skipif( not pytest_options.easlrb_path, reason="eASLRB not available." ) |
||||
def test_full_prepare( webapp, webdriver ): |
||||
"""Test the full prepare process.""" |
||||
|
||||
def do_test( dname ): |
||||
|
||||
# initialize |
||||
# NOTE: We load the webapp without setting a data directory first. |
||||
init_webapp( webapp, webdriver ) |
||||
|
||||
# load the PDF file data into the web page (since we can't manipulate the "Open File" dialog) |
||||
fname = os.path.join( dname, "eASLRB.pdf" ) |
||||
with open( fname, "rb" ) as fp: |
||||
zip_data = fp.read() |
||||
testing_zip_data = find_child( "#testing-zip-data", webdriver ) |
||||
webdriver.execute_script( "arguments[0].value = arguments[1]", testing_zip_data, |
||||
base64.b64encode( zip_data ).decode( "ascii" ) |
||||
) |
||||
|
||||
# start the prepare process, and wait for it to finish |
||||
# NOTE: It will have auto-started because we passed in a filename to the webapp. |
||||
find_child( "button#upload-proxy" ).click() |
||||
wait_for_elem( 30*60, "#download-panel" ) |
||||
|
||||
# get the results |
||||
find_child( "button#download" ).click() |
||||
zip_data = wait_for( 20, lambda: testing_zip_data.get_attribute( "value" ) ) |
||||
zip_data = base64.b64decode( zip_data ) |
||||
|
||||
# check the results |
||||
with zipfile.ZipFile( io.BytesIO( zip_data ) ) as zip_file: |
||||
assert set( zip_file.namelist() ) == set( [ |
||||
"ASL Rulebook.pdf", "ASL Rulebook.index", |
||||
"ASL Rulebook.targets", "ASL Rulebook.chapters", "ASL Rulebook.footnotes" |
||||
] ) |
||||
assert zip_file.getinfo( "ASL Rulebook.pdf" ).file_size > 40*1000 |
||||
for ftype in [ "index", "targets", "chapters", "footnotes" ]: |
||||
fname = os.path.join( dname, ftype+".json" ) |
||||
expected = json.load( open( fname, "r" ) ) |
||||
fdata = zip_file.read( "ASL Rulebook.{}".format( ftype ) ) |
||||
assert json.loads( fdata ) == expected |
||||
|
||||
# run the test |
||||
for_each_easlrb_version( do_test ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _unload_progress(): |
||||
"""Unload the progress messages.""" |
||||
|
||||
def unload_status_block( root ): |
||||
"""Unload a status block and its progress messages.""" |
||||
caption = find_child( ".caption", root ).text |
||||
msgs = [ |
||||
unload_msg( row ) |
||||
for row in find_children( "tr", root ) |
||||
if row.is_displayed() |
||||
] |
||||
return [ caption, msgs ] |
||||
|
||||
def unload_msg( row ): |
||||
"""Unload a single progress message.""" |
||||
cells = find_children( "td", row ) |
||||
assert len(cells) == 2 |
||||
img = find_child( "img", cells[0] ) |
||||
url = img.get_attribute( "src" ) |
||||
return [ os.path.basename(url), cells[1].text ] |
||||
|
||||
# unload each status block |
||||
progress_panel = find_child( "#progress-panel" ) |
||||
return [ |
||||
unload_status_block( elem ) |
||||
for elem in find_children( ".status", progress_panel ) |
||||
] |
||||
Loading…
Reference in new issue