|
|
|
""" Manage the startup process. """
|
|
|
|
|
|
|
|
import os
|
|
|
|
import time
|
|
|
|
import datetime
|
|
|
|
import threading
|
|
|
|
import logging
|
|
|
|
import traceback
|
|
|
|
import enum
|
|
|
|
from collections import defaultdict
|
|
|
|
|
|
|
|
from flask import jsonify
|
|
|
|
|
|
|
|
from asl_rulebook2.webapp import app
|
|
|
|
from asl_rulebook2.webapp.content import load_content_sets
|
|
|
|
from asl_rulebook2.webapp.search import init_search
|
|
|
|
from asl_rulebook2.webapp.rule_info import init_qa, init_errata, init_annotations
|
|
|
|
from asl_rulebook2.webapp.asop import init_asop
|
|
|
|
from asl_rulebook2.webapp.utils import parse_int
|
|
|
|
|
|
|
|
_capabilities = None
|
|
|
|
|
|
|
|
_startup_tasks = None
|
|
|
|
|
|
|
|
_logger = logging.getLogger( "startup" )
|
|
|
|
_startup_msgs = None
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
class StartupStatusEnum( enum.IntEnum ): #pylint: disable=missing-class-docstring
|
|
|
|
NOT_STARTED = 0
|
|
|
|
STARTED = 1
|
|
|
|
TASKS_RUNNING = 2
|
|
|
|
COMPLETED = -1
|
|
|
|
|
|
|
|
_startup_status = StartupStatusEnum.NOT_STARTED
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
def init_webapp():
|
|
|
|
"""Initialize the webapp.
|
|
|
|
|
|
|
|
IMPORTANT: This is called on the first Flask request, but can also be called multiple times
|
|
|
|
after that by the test suite, to reset the webapp before each test.
|
|
|
|
"""
|
|
|
|
|
|
|
|
# initialize
|
|
|
|
global _startup_status, _startup_msgs, _capabilities, _startup_tasks
|
|
|
|
_startup_status = StartupStatusEnum.STARTED
|
|
|
|
_startup_msgs = StartupMsgs()
|
|
|
|
_capabilities = {}
|
|
|
|
_startup_tasks = []
|
|
|
|
|
|
|
|
# initialize the webapp
|
|
|
|
content_sets = load_content_sets( _startup_msgs, _logger )
|
|
|
|
if content_sets:
|
|
|
|
_capabilities[ "content-sets" ] = True
|
|
|
|
qa, qa_fnames = init_qa( _startup_msgs, _logger )
|
|
|
|
if qa:
|
|
|
|
_capabilities[ "qa" ] = True
|
|
|
|
errata, errata_fnames = init_errata( _startup_msgs, _logger )
|
|
|
|
if errata:
|
|
|
|
_capabilities[ "errata" ] = True
|
|
|
|
user_anno, user_anno_fname = init_annotations( _startup_msgs, _logger )
|
|
|
|
if user_anno:
|
|
|
|
_capabilities[ "user-anno" ] = True
|
|
|
|
asop, asop_preambles, asop_content, asop_fnames = init_asop( _startup_msgs, _logger )
|
|
|
|
if asop:
|
|
|
|
_capabilities[ "asop" ] = True
|
|
|
|
init_search(
|
|
|
|
content_sets,
|
|
|
|
qa, qa_fnames,
|
|
|
|
errata, errata_fnames,
|
|
|
|
user_anno, user_anno_fname,
|
|
|
|
asop, asop_preambles, asop_content, asop_fnames,
|
|
|
|
_startup_msgs, _logger
|
|
|
|
)
|
|
|
|
|
|
|
|
# everything has been initialized - now we can go back and fixup content
|
|
|
|
# NOTE: This is quite a slow process (~1 minute for a full data load), which is why we don't do it inline,
|
|
|
|
# during the normal startup process. So, we start up using the original content, and if the user does
|
|
|
|
# a search, that's what they will see, but we fix it up in the background, and the new content will
|
|
|
|
# eventually start to be returned as search results.
|
|
|
|
if app.config.get( "BLOCKING_STARTUP_TASKS" ):
|
|
|
|
# NOTE: It's useful to do this synchronously when running the test suite, since if the tests
|
|
|
|
# need the linkified ruleid's, they can't start until the fixup has finished (and if they don't
|
|
|
|
# it won't really matter, since there will be so little data, this process will be fast).
|
|
|
|
_do_startup_tasks( False )
|
|
|
|
else:
|
|
|
|
threading.Thread( target=_do_startup_tasks, args=(True,) ).start()
|
|
|
|
|
|
|
|
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
|
|
|
|
|
|
|
def _add_startup_task( ctype, func ):
|
|
|
|
"""Register a function to run at startup."""
|
|
|
|
if app.config.get( "DISABLE_STARTUP_TASKS" ):
|
|
|
|
return
|
|
|
|
_startup_tasks.append( ( ctype, func ) )
|
|
|
|
|
|
|
|
def _do_startup_tasks( delay ):
|
|
|
|
"""Run each registered startup task."""
|
|
|
|
|
|
|
|
# initialize
|
|
|
|
global _startup_status
|
|
|
|
if not _startup_tasks:
|
|
|
|
_startup_status = StartupStatusEnum.COMPLETED
|
|
|
|
return
|
|
|
|
|
|
|
|
# FUDGE! If we start processing straight away, the main PDF loads very slowly because of us :-/,
|
|
|
|
# and since there's no way to set thread priorities in Python, we delay for a short time, to give
|
|
|
|
# the PDF time to load, before we start working.
|
|
|
|
# NOTE: This delay only helps the initial load of the main ASLRB PDF. After processing has started,
|
|
|
|
# if the user reloads the page, or tries to load another PDF, they will have the same problem of
|
|
|
|
# very slow loads. To work around this, _tag_ruleids_in_field() sleeps periodically, to give
|
|
|
|
# other threads a chance to run. The PDF's load a bit slowly, but it's acceptable.
|
|
|
|
# NOTE: If there is a cached search database, things are very fast and so we don't need to delay.
|
|
|
|
fname = app.config.get( "CACHED_SEARCHDB" )
|
|
|
|
have_cached_searchdb = fname and os.path.isfile( fname ) and os.path.getsize( fname ) > 0
|
|
|
|
if delay and not have_cached_searchdb:
|
|
|
|
delay = parse_int( app.config.get( "STARTUP_TASKS_DELAY" ), 5 )
|
|
|
|
time.sleep( delay )
|
|
|
|
|
|
|
|
# process each startup task
|
|
|
|
_startup_status = StartupStatusEnum.TASKS_RUNNING
|
|
|
|
_logger.info( "Processing startup tasks..." )
|
|
|
|
start_time = time.time()
|
|
|
|
for task_no, (ctype, func) in enumerate( _startup_tasks ):
|
|
|
|
_logger.debug( "Running startup task (%d/%d): %s", 1+task_no, len(_startup_tasks), ctype )
|
|
|
|
start_time2 = time.time()
|
|
|
|
try:
|
|
|
|
msg = func()
|
|
|
|
except Exception as ex: #pylint: disable=broad-except
|
|
|
|
_logger.error( "Startup task '%s' failed: %s\n%s", ctype, ex, traceback.format_exc() )
|
|
|
|
continue
|
|
|
|
elapsed_time = datetime.timedelta( seconds = int( time.time() - start_time2 ) )
|
|
|
|
msg = ": {}".format( msg ) if msg else "."
|
|
|
|
_logger.debug( "- Finished startup task (%s)%s", elapsed_time, msg )
|
|
|
|
|
|
|
|
# finish up
|
|
|
|
elapsed_time = datetime.timedelta( seconds = int( time.time() - start_time ) )
|
|
|
|
_logger.info( "All startup tasks completed (%s).", elapsed_time )
|
|
|
|
_startup_status = StartupStatusEnum.COMPLETED
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
@app.route( "/app-config" )
|
|
|
|
def get_app_config():
|
|
|
|
"""Return the app config."""
|
|
|
|
|
|
|
|
# initialize
|
|
|
|
_logger.debug( "Sending app config:" )
|
|
|
|
result = {}
|
|
|
|
|
|
|
|
# send the available capabilities
|
|
|
|
_logger.debug( "- capabilities: %s", _capabilities )
|
|
|
|
result["capabilities"] = _capabilities
|
|
|
|
|
|
|
|
# send any user-defined debug settings
|
|
|
|
for key in app.config:
|
|
|
|
if not key.startswith( "WEBAPP_" ):
|
|
|
|
continue
|
|
|
|
val = app.config.get( key )
|
|
|
|
if val is not None:
|
|
|
|
_logger.debug( "- %s = %s", key, val )
|
|
|
|
result[ key ] = parse_int( val, val )
|
|
|
|
|
|
|
|
return jsonify( result )
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
@app.route( "/startup-msgs" )
|
|
|
|
def get_startup_msgs():
|
|
|
|
"""Return any messages issued during startup."""
|
|
|
|
return jsonify( _startup_msgs.msgs )
|
|
|
|
|
|
|
|
@app.route( "/startup-status" )
|
|
|
|
def get_startup_status():
|
|
|
|
"""Return the current startup status."""
|
|
|
|
return jsonify( {
|
|
|
|
"status": _startup_status
|
|
|
|
} )
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
|
|
class StartupMsgs:
|
|
|
|
"""Store messages issued during startup."""
|
|
|
|
|
|
|
|
def __init__( self ):
|
|
|
|
self.msgs = defaultdict( list )
|
|
|
|
|
|
|
|
#pylint: disable=missing-function-docstring
|
|
|
|
def info( self, msg, msg_info=None ):
|
|
|
|
return self._add_msg( "info", msg, msg_info )
|
|
|
|
def warning( self, msg, msg_info=None ):
|
|
|
|
return self._add_msg( "warning", msg, msg_info )
|
|
|
|
def error( self, msg, msg_info=None ):
|
|
|
|
return self._add_msg( "error", msg, msg_info )
|
|
|
|
|
|
|
|
def _add_msg( self, msg_type, msg, msg_info ):
|
|
|
|
"""Add a startup message."""
|
|
|
|
if msg_info:
|
|
|
|
self.msgs[ msg_type ].append( ( msg, msg_info ) )
|
|
|
|
getattr( _logger, msg_type )( "%s\n %s", msg, msg_info )
|
|
|
|
else:
|
|
|
|
self.msgs[ msg_type ].append( msg )
|
|
|
|
getattr( _logger, msg_type )( "%s", msg )
|