asl-rulebook2/asl_rulebook2/webapp/startup.py

""" Manage the startup process. """

import os
import time
import datetime
import threading
import logging
import traceback
import enum
from collections import defaultdict

from flask import jsonify

from asl_rulebook2.webapp import app
from asl_rulebook2.webapp.content import load_content_sets
from asl_rulebook2.webapp.search import init_search
from asl_rulebook2.webapp.rule_info import init_qa, init_errata, init_annotations
from asl_rulebook2.webapp.asop import init_asop
from asl_rulebook2.webapp.utils import parse_int

_capabilities = None

_startup_tasks = None

_logger = logging.getLogger( "startup" )
_startup_msgs = None

# ---------------------------------------------------------------------

class StartupStatusEnum( enum.IntEnum ): #pylint: disable=missing-class-docstring
    NOT_STARTED = 0
    STARTED = 1
    TASKS_RUNNING = 2
    COMPLETED = -1

_startup_status = StartupStatusEnum.NOT_STARTED

# ---------------------------------------------------------------------

def init_webapp():
    """Initialize the webapp.

    IMPORTANT: This is called on the first Flask request, but can also be called multiple times
    after that by the test suite, to reset the webapp before each test.
    """

    # initialize
    global _startup_status, _startup_msgs, _capabilities, _startup_tasks
    _startup_status = StartupStatusEnum.STARTED
    _startup_msgs = StartupMsgs()
    _capabilities = {}
    _startup_tasks = []

    # initialize the webapp
    content_sets = load_content_sets( _startup_msgs, _logger )
    if content_sets:
        _capabilities[ "content-sets" ] = True
    qa, qa_fnames = init_qa( _startup_msgs, _logger )
    if qa:
        _capabilities[ "qa" ] = True
    errata, errata_fnames = init_errata( _startup_msgs, _logger )
    if errata:
        _capabilities[ "errata" ] = True
    user_anno, user_anno_fname = init_annotations( _startup_msgs, _logger )
    if user_anno:
        _capabilities[ "user-anno" ] = True
    asop, asop_preambles, asop_content, asop_fnames = init_asop( _startup_msgs, _logger )
    if asop:
        _capabilities[ "asop" ] = True
    init_search(
        content_sets,
        qa, qa_fnames,
        errata, errata_fnames,
        user_anno, user_anno_fname,
        asop, asop_preambles, asop_content, asop_fnames,
        _startup_msgs, _logger
    )

    # everything has been initialized - now we can go back and fixup content
    # NOTE: This is quite a slow process (~1 minute for a full data load), which is why we don't do it inline,
    # during the normal startup process. So, we start up using the original content, and if the user does
    # a search, that's what they will see, but we fix it up in the background, and the new content will
    # eventually start to be returned as search results.
    if app.config.get( "BLOCKING_STARTUP_TASKS" ):
        # NOTE: It's useful to do this synchronously when running the test suite, since if the tests
        # need the linkified ruleid's, they can't start until the fixup has finished (and if they don't
        # it won't really matter, since there will be so little data, this process will be fast).
        _do_startup_tasks( False )
    else:
        threading.Thread( target=_do_startup_tasks, args=(True,) ).start()

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

def _add_startup_task( ctype, func ):
    """Register a function to run at startup."""
    if app.config.get( "DISABLE_STARTUP_TASKS" ):
        return
    _startup_tasks.append( ( ctype, func ) )

def _do_startup_tasks( delay ):
    """Run each registered startup task."""

    # initialize
    global _startup_status
    if not _startup_tasks:
        _startup_status = StartupStatusEnum.COMPLETED
        return

    # FUDGE! If we start processing straight away, the main PDF loads very slowly because of us :-/,
    # and since there's no way to set thread priorities in Python, we delay for a short time, to give
    # the PDF time to load, before we start working.
    # NOTE: This delay only helps the initial load of the main ASLRB PDF. After processing has started,
    # if the user reloads the page, or tries to load another PDF, they will have the same problem of
    # very slow loads. To work around this, _tag_ruleids_in_field() sleeps periodically, to give
    # other threads a chance to run. The PDF's load a bit slowly, but it's acceptable.
    # NOTE: If there is a cached search database, things are very fast and so we don't need to delay.
    fname = app.config.get( "CACHED_SEARCHDB" )
    have_cached_searchdb = fname and os.path.isfile( fname ) and os.path.getsize( fname ) > 0
    if delay and not have_cached_searchdb:
        delay = parse_int( app.config.get( "STARTUP_TASKS_DELAY" ), 5 )
        time.sleep( delay )

    # process each startup task
    _startup_status = StartupStatusEnum.TASKS_RUNNING
    _logger.info( "Processing startup tasks..." )
    start_time = time.time()
    for task_no, (ctype, func) in enumerate( _startup_tasks ):
        _logger.debug( "Running startup task (%d/%d): %s", 1+task_no, len(_startup_tasks), ctype )
        start_time2 = time.time()
        try:
            msg = func()
        except Exception as ex: #pylint: disable=broad-except
            _logger.error( "Startup task '%s' failed: %s\n%s", ctype, ex, traceback.format_exc() )
            continue
        elapsed_time = datetime.timedelta( seconds = int( time.time() - start_time2 ) )
        msg = ": {}".format( msg ) if msg else "."
        _logger.debug( "- Finished startup task (%s)%s", elapsed_time, msg )

    # finish up
    elapsed_time = datetime.timedelta( seconds = int( time.time() - start_time ) )
    _logger.info( "All startup tasks completed (%s).", elapsed_time )
    _startup_status = StartupStatusEnum.COMPLETED

# ---------------------------------------------------------------------

@app.route( "/app-config" )
def get_app_config():
    """Return the app config."""

    # initialize
    _logger.debug( "Sending app config:" )
    result = {}

    # send the available capabilities
    _logger.debug( "- capabilities: %s", _capabilities )
    result["capabilities"] = _capabilities

    # send any user-defined debug settings
    for key in app.config:
        if not key.startswith( "WEBAPP_" ):
            continue
        val = app.config.get( key )
        if val is not None:
            _logger.debug( "- %s = %s", key, val )
            result[ key ] = parse_int( val, val )

    return jsonify( result )

# ---------------------------------------------------------------------

@app.route( "/startup-msgs" )
def get_startup_msgs():
    """Return any messages issued during startup."""
    return jsonify( _startup_msgs.msgs )

@app.route( "/startup-status" )
def get_startup_status():
    """Return the current startup status."""
    return jsonify( {
        "status": _startup_status
    } )

# ---------------------------------------------------------------------

class StartupMsgs:
    """Store messages issued during startup."""

    def __init__( self ):
        self.msgs = defaultdict( list )

    #pylint: disable=missing-function-docstring
    def info( self, msg, msg_info=None ):
        return self._add_msg( "info", msg, msg_info )
    def warning( self, msg, msg_info=None ):
        return self._add_msg( "warning", msg, msg_info )
    def error( self, msg, msg_info=None ):
        return self._add_msg( "error", msg, msg_info )

    def _add_msg( self, msg_type, msg, msg_info ):
        """Add a startup message."""
        if msg_info:
            self.msgs[ msg_type ].append( ( msg, msg_info ) )
            getattr( _logger, msg_type )( "%s\n  %s", msg, msg_info )
        else:
            self.msgs[ msg_type ].append( msg )
            getattr( _logger, msg_type )( "%s", msg )