From f080805e77e68e6c623acaf03b2eb7abfcf9e9d7 Mon Sep 17 00:00:00 2001 From: Taka Date: Thu, 25 Nov 2021 12:24:52 +1100 Subject: [PATCH] Fixed some issues when running the test suite against a remote backend server. --- asl_articles/__init__.py | 4 +-- asl_articles/search.py | 38 ++++++++++++++++++---------- asl_articles/tests/test_db_report.py | 14 ++++++++++ asl_articles/tests/test_startup.py | 2 +- asl_articles/tests/utils.py | 11 ++++---- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/asl_articles/__init__.py b/asl_articles/__init__.py index 2d3e878..fd1ba88 100644 --- a/asl_articles/__init__.py +++ b/asl_articles/__init__.py @@ -40,8 +40,8 @@ def _on_startup(): return # initialize the search index - _logger = logging.getLogger( "startup" ) - asl_articles.search.init_search( db.session, _logger ) + logger = logging.getLogger( "startup" ) + asl_articles.search.init_search( db.session, logger ) # --------------------------------------------------------------------- diff --git a/asl_articles/search.py b/asl_articles/search.py index 9c8c353..c45d9ef 100644 --- a/asl_articles/search.py +++ b/asl_articles/search.py @@ -568,7 +568,7 @@ def _find_aslrb_ruleids( val ): #pylint: disable=too-many-branches # --------------------------------------------------------------------- -def init_search( session, logger ): +def init_search( session, logger, test_mode=False ): """Initialize the search engine.""" # initialize the database @@ -612,20 +612,22 @@ def init_search( session, logger ): for article in session.query( Article ).order_by( Article.time_created.desc() ): add_or_update_article( dbconn, article, session ) - # configure the searcg engine + # configure the search engine + global _search_aliases + _search_aliases = {} + global _search_weights + _search_weights = {} fname = os.path.join( asl_articles.config_dir, "search.cfg" ) if os.path.isfile( fname ): # load the search aliases _logger.debug( "Loading search aliases: %s", fname ) cfg = AppConfigParser( fname ) - global _search_aliases _search_aliases = _load_search_aliases( cfg.get_section( "Search aliases" ), cfg.get_section( "Search aliases 2" ) ) # load the search weights _logger.debug( "Loading search weights:" ) - global _search_weights for row in cfg.get_section( "Search weights" ): if row[0] not in _SEARCHABLE_COL_NAMES: asl_articles.startup.log_startup_msg( "warning", @@ -646,20 +648,22 @@ def init_search( session, logger ): # NOTE: These should really be stored in the database, but the UI would be so insanely hairy, # we just keep them in a text file and let the user manage them manually :-/ global _author_aliases + _author_aliases = {} fname = os.path.join( asl_articles.config_dir, "author-aliases.cfg" ) if os.path.isfile( fname ): _logger.debug( "Loading author aliases: %s", fname ) cfg = AppConfigParser( fname ) _author_aliases = _load_author_aliases( cfg.get_section("Author aliases"), session, False ) - # NOTE: We load the test aliases here as well (the test suite can't mock them because - # they might be running in a different process). - fname = os.path.join( os.path.split(__file__)[0], "tests/fixtures/author-aliases.cfg" ) - if os.path.isfile( fname ): - _logger.debug( "Loading test author aliases: %s", fname ) - cfg = AppConfigParser( fname ) - _author_aliases.update( - _load_author_aliases( cfg.get_section("Author aliases"), session, True ) - ) + if test_mode: + # NOTE: We load the test aliases here as well (since the test suite can't mock them, + # because we might be running in a different process). + fname = os.path.join( os.path.split(__file__)[0], "tests/fixtures/author-aliases.cfg" ) + if os.path.isfile( fname ): + _logger.debug( "Loading test author aliases: %s", fname ) + cfg = AppConfigParser( fname ) + _author_aliases.update( + _load_author_aliases( cfg.get_section("Author aliases"), session, True ) + ) def _load_search_aliases( aliases, aliases2 ): """Load the search aliases.""" @@ -827,3 +831,11 @@ def _make_publication_key( pub ): def _make_article_key( article ): """Generate the owner key for an Article.""" return "article:{}".format( article.article_id if isinstance(article,Article) else article ) + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +@app.route( "/init-search-for-test" ) +def init_search_for_test(): + """Re-initialize the search engine (for testing porpoises).""" + init_search( db.session, logging.getLogger("search"), test_mode=True ) + return "ok" diff --git a/asl_articles/tests/test_db_report.py b/asl_articles/tests/test_db_report.py index 9782a7a..a033adc 100644 --- a/asl_articles/tests/test_db_report.py +++ b/asl_articles/tests/test_db_report.py @@ -4,6 +4,8 @@ import os import itertools import re +import pytest + from asl_articles.search import SEARCH_ALL from asl_articles.tests.test_publishers import edit_publisher from asl_articles.tests.test_publications import edit_publication @@ -12,6 +14,7 @@ from asl_articles.tests.utils import init_tests, \ select_main_menu_option, select_sr_menu_option, check_ask_dialog, \ do_search, find_search_result, get_search_results, \ wait_for, wait_for_elem, find_child, find_children +from asl_articles.tests import pytest_options # --------------------------------------------------------------------- @@ -102,6 +105,17 @@ def test_db_report( webdriver, flask_app, dbconn ): # --------------------------------------------------------------------- +# NOTE: This test may not work if we are running against Docker containers, because: +# - external URL's are created that point to the back-end's $/ping endpoint. +# - the front-end container realizes that these URL's need to be checked by the backend, +# so it sends them to the $/db-report/check-link endpoint. +# - these URL's may not resolve because they were generated using gAppRef.makeFlaskUrl(), +# which will work if the front-end container is sending a request to the back-end +# container, but may not work from inside the back-end container, because the port number +# being used by Flask *inside* the container may not be the same as *outside* the container. +# The problem is generating a URL that can be used as an external URL that will work everywhere. +# We could specify it as a parameter, but that's more trouble than it's worth. +@pytest.mark.skipif( pytest_options.flask_url is not None, reason="Testing against a remote Flask server." ) def test_check_db_links( webdriver, flask_app, dbconn ): """Test checking links in the database.""" diff --git a/asl_articles/tests/test_startup.py b/asl_articles/tests/test_startup.py index dbe61c0..7c209f2 100644 --- a/asl_articles/tests/test_startup.py +++ b/asl_articles/tests/test_startup.py @@ -9,7 +9,7 @@ from asl_articles.tests import pytest_options # --------------------------------------------------------------------- -@pytest.mark.skipif( pytest_options.flask_url is not None, reason="Testing against a remote Flask server." ) +@pytest.mark.skipif( pytest_options.flask_url is not None, reason="Testing against a remote Flask server." ) def test_startup_messages( webdriver, flask_app, dbconn ): """Test startup messages.""" diff --git a/asl_articles/tests/utils.py b/asl_articles/tests/utils.py index 60420c3..e002bce 100644 --- a/asl_articles/tests/utils.py +++ b/asl_articles/tests/utils.py @@ -1,12 +1,12 @@ """ Helper utilities for the test suite. """ import os +import urllib.request import json import time import itertools import uuid import base64 -import logging import sqlalchemy import sqlalchemy.orm @@ -19,7 +19,6 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutException -from asl_articles import search from asl_articles.utils import to_bool import asl_articles.models @@ -47,6 +46,11 @@ def init_tests( webdriver, flask_app, dbconn, **kwargs ): assert fixtures is None session = None + # re-initialize the search engine + if flask_app: + url = flask_app.url_for( "init_search_for_test" ) + _ = urllib.request.urlopen( url ).read() + # initialize the documents directory dname = kwargs.pop( "docs", None ) if dname: @@ -95,9 +99,6 @@ def load_fixtures( session, fname ): session.bulk_insert_mappings( model, data[table_name] ) session.commit() - # rebuild the search index - search.init_search( session, logging.getLogger("search") ) - # --------------------------------------------------------------------- def do_search( query ):