parent
f60986d478
commit
782bebadca
@ -0,0 +1,45 @@ |
||||
"""Added the 'scenario' tables. |
||||
|
||||
Revision ID: e77d4e8d37f3 |
||||
Revises: 23e928dda837 |
||||
Create Date: 2019-12-20 09:46:35.304849 |
||||
|
||||
""" |
||||
from alembic import op |
||||
import sqlalchemy as sa |
||||
|
||||
|
||||
# revision identifiers, used by Alembic. |
||||
revision = 'e77d4e8d37f3' |
||||
down_revision = '23e928dda837' |
||||
branch_labels = None |
||||
depends_on = None |
||||
|
||||
|
||||
def upgrade(): |
||||
# ### commands auto generated by Alembic - please adjust! ### |
||||
op.create_table('scenario', |
||||
sa.Column('scenario_id', sa.Integer(), nullable=False), |
||||
sa.Column('scenario_roar_id', sa.String(length=50), nullable=True), |
||||
sa.Column('scenario_display_id', sa.String(length=50), nullable=True), |
||||
sa.Column('scenario_name', sa.String(length=200), nullable=False), |
||||
sa.PrimaryKeyConstraint('scenario_id'), |
||||
sa.UniqueConstraint('scenario_roar_id', 'scenario_display_id', 'scenario_name', name='unq_id_name') |
||||
) |
||||
op.create_table('article_scenario', |
||||
sa.Column('article_scenario_id', sa.Integer(), nullable=False), |
||||
sa.Column('seq_no', sa.Integer(), nullable=False), |
||||
sa.Column('article_id', sa.Integer(), nullable=False), |
||||
sa.Column('scenario_id', sa.Integer(), nullable=False), |
||||
sa.ForeignKeyConstraint(['article_id'], ['article.article_id'], ondelete='CASCADE'), |
||||
sa.ForeignKeyConstraint(['scenario_id'], ['scenario.scenario_id'], ondelete='CASCADE'), |
||||
sa.PrimaryKeyConstraint('article_scenario_id') |
||||
) |
||||
# ### end Alembic commands ### |
||||
|
||||
|
||||
def downgrade(): |
||||
# ### commands auto generated by Alembic - please adjust! ### |
||||
op.drop_table('article_scenario') |
||||
op.drop_table('scenario') |
||||
# ### end Alembic commands ### |
||||
@ -0,0 +1,28 @@ |
||||
""" Handle scenario requests. """ |
||||
|
||||
from flask import jsonify |
||||
|
||||
from asl_articles import app |
||||
from asl_articles.models import Scenario |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/scenarios" ) |
||||
def get_scenarios(): |
||||
"""Get all scenarios.""" |
||||
return jsonify( do_get_scenarios() ) |
||||
|
||||
def do_get_scenarios(): |
||||
"""Get all scenarios.""" |
||||
return { |
||||
s.scenario_id: _get_scenario_vals( s ) |
||||
for s in Scenario.query #pylint: disable=not-an-iterable |
||||
} |
||||
|
||||
def _get_scenario_vals( scenario ): |
||||
"""Extract public fields from a scenario record.""" |
||||
return { |
||||
"scenario_id": scenario.scenario_id, |
||||
"scenario_display_id": scenario.scenario_display_id, |
||||
"scenario_name": scenario.scenario_name |
||||
} |
||||
@ -0,0 +1,10 @@ |
||||
{ |
||||
|
||||
"scenario": [ |
||||
{ "scenario_display_id": "TEST 1", "scenario_name": "Test Scenario 1" }, |
||||
{ "scenario_display_id": "TEST 2", "scenario_name": "Test Scenario 2" }, |
||||
{ "scenario_display_id": "TEST 3", "scenario_name": "Test Scenario 3" }, |
||||
{ "scenario_name": "No scenario ID" } |
||||
] |
||||
|
||||
} |
||||
@ -0,0 +1,7 @@ |
||||
{ |
||||
|
||||
"1": { "scenario_id": "1", "name": "Fighting Withdrawal" }, |
||||
"99": { "scenario_id": "FOO BAR", "name": "test scenario" }, |
||||
"129": { "scenario_id": "E", "name": "Hill 621" } |
||||
|
||||
} |
||||
@ -0,0 +1,131 @@ |
||||
""" Test importing ROAR scenarios into our database. """ |
||||
|
||||
import sys |
||||
import os |
||||
import json |
||||
|
||||
from asl_articles.models import Scenario |
||||
from asl_articles.tests.utils import init_db |
||||
|
||||
sys.path.append( os.path.join( os.path.split(__file__)[0], "../../tools/" ) ) |
||||
from import_roar_scenarios import import_roar_scenarios |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def test_import_roar_scenarios( dbconn ): |
||||
"""Test importing ROAR scenarios.""" |
||||
|
||||
# initialize |
||||
session = init_db( dbconn, None ) |
||||
roar_fname = os.path.join( os.path.split(__file__)[0], "fixtures/roar-scenarios.json" ) |
||||
roar_data = json.load( open( roar_fname, "r" ) ) |
||||
|
||||
# do the first import |
||||
_do_import( dbconn, session, roar_fname, |
||||
{ "nInserts": 3, "nUpdates": 0, "nDupes": 0 }, [ |
||||
[ "1", "1", "Fighting Withdrawal" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ] |
||||
] ) |
||||
|
||||
# repeat the import (nothing should happen) |
||||
_do_import( dbconn, session, roar_fname, |
||||
{ "nInserts": 0, "nUpdates": 0, "nDupes": 3 }, [ |
||||
[ "1", "1", "Fighting Withdrawal" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ] |
||||
] ) |
||||
|
||||
# simulate a scenario's details being updated in ROAR (we should update accordingly) |
||||
roar_data["1"]["scenario_id"] += "u" |
||||
roar_data["1"]["name"] += " (updated)" |
||||
_do_import( dbconn, session, roar_data, |
||||
{ "nInserts": 0, "nUpdates": 1, "nDupes": 2 }, [ |
||||
[ "1", "1u", "Fighting Withdrawal (updated)" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ], |
||||
], |
||||
[ "Data mismatch for ROAR ID 1:" ] |
||||
) |
||||
|
||||
# add a new ROAR scenario (we should import the new scenario) |
||||
roar_data[ "42" ] = { "scenario_id": "NEW", "name": "new scenario" } |
||||
_do_import( dbconn, session, roar_data, |
||||
{ "nInserts": 1, "nUpdates": 0, "nDupes": 3 }, [ |
||||
[ "1", "1u", "Fighting Withdrawal (updated)" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ], |
||||
[ "42", "NEW", "new scenario" ] |
||||
] ) |
||||
|
||||
# delete all ROAR scenarios (nothing should happen) |
||||
_do_import( dbconn, session, {}, |
||||
{ "nInserts": 0, "nUpdates": 0, "nDupes": 0 }, [ |
||||
[ "1", "1u", "Fighting Withdrawal (updated)" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ], |
||||
[ "42", "NEW", "new scenario" ] |
||||
] ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def test_scenario_matching( dbconn ): |
||||
"""Test matching ROAR scenarios with scenarios in the database.""" |
||||
|
||||
# initialize |
||||
session = init_db( dbconn, None ) |
||||
roar_fname = os.path.join( os.path.split(__file__)[0], "fixtures/roar-scenarios.json" ) |
||||
|
||||
# put a scenario in the database that has no ROAR ID |
||||
session.add( Scenario( scenario_display_id="1", scenario_name="Fighting Withdrawal" ) ) |
||||
session.commit() |
||||
|
||||
# do an import |
||||
# NOTE: The scenario we created above will be matched to the corresponding ROAR scenario, |
||||
# since the ROAR ID is not considered when matching scenarios. |
||||
_do_import( dbconn, session, roar_fname, |
||||
{ "nInserts": 2, "nUpdates": 0, "nDupes": 1 }, [ |
||||
[ None, "1", "Fighting Withdrawal" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ "129", "E", "Hill 621" ] |
||||
] ) |
||||
|
||||
# put a scenario in the database that only has a name |
||||
session.query( Scenario ).delete() |
||||
session.add( Scenario( scenario_name="Hill 621" ) ) |
||||
session.commit() |
||||
|
||||
# do an import |
||||
# NOTE: The scenario we created above will be matched to the corresponding ROAR scenario, |
||||
# and also updated with the scenario ID (as reported by ROAR). |
||||
_do_import( dbconn, session, roar_fname, |
||||
{ "nInserts": 2, "nUpdates": 1, "nDupes": 0 }, [ |
||||
[ "1", "1", "Fighting Withdrawal" ], |
||||
[ "99", "FOO BAR", "test scenario" ], |
||||
[ None, "E", "Hill 621" ] |
||||
], |
||||
[ "Data mismatch for ROAR ID 129:" ] |
||||
) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _do_import( dbconn, session, roar_data, expected_stats, expected_scenarios, expected_warnings=None ): |
||||
"""Import the ROAR scenarios and check the results.""" |
||||
|
||||
# import the ROAR scenarios |
||||
stats, warnings = import_roar_scenarios( dbconn.url, roar_data ) |
||||
|
||||
# check that the import went as expected |
||||
assert stats == expected_stats |
||||
if expected_warnings: |
||||
assert warnings[0][0] == expected_warnings[0] |
||||
else: |
||||
assert not warnings |
||||
|
||||
# check the scenarios in the database |
||||
rows = [ |
||||
[ s.scenario_roar_id, s.scenario_display_id, s.scenario_name ] |
||||
for s in session.query( Scenario ) #pylint: disable=not-an-iterable |
||||
] |
||||
sort_rows = lambda rows: sorted( rows, key=lambda r: r[2] ) |
||||
assert sort_rows( rows ) == sort_rows( expected_scenarios ) |
||||
@ -0,0 +1,102 @@ |
||||
""" Test article scenario operations. """ |
||||
|
||||
import urllib.request |
||||
import json |
||||
|
||||
from asl_articles.tests.utils import init_tests, find_child, wait_for_elem, find_search_result |
||||
from asl_articles.tests.react_select import ReactSelect |
||||
|
||||
from asl_articles.tests.test_articles import _create_article, _edit_article |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def test_article_scenarios( webdriver, flask_app, dbconn ): |
||||
"""Test article scenario operations.""" |
||||
|
||||
# initialize |
||||
init_tests( webdriver, flask_app, dbconn, fixtures="article-scenarios.json" ) |
||||
all_scenarios = set( [ |
||||
"Test Scenario 1 [TEST 1]", "Test Scenario 2 [TEST 2]", "Test Scenario 3 [TEST 3]", |
||||
"No scenario ID" |
||||
] ) |
||||
|
||||
# create some test articles |
||||
_create_article( { "title": "article 1" } ) |
||||
_create_article( { "title": "article 2" } ) |
||||
_check_scenarios( flask_app, all_scenarios, [ [], [] ] ) |
||||
|
||||
# add a scenario to article #1 |
||||
_edit_article( find_search_result( "article 1" ), { |
||||
"scenarios": [ "+Test Scenario 1 [TEST 1]" ] |
||||
} ) |
||||
_check_scenarios( flask_app, all_scenarios, [ |
||||
[ "Test Scenario 1 [TEST 1]" ], |
||||
[] |
||||
] ) |
||||
|
||||
# add scenarios to article #2 |
||||
_edit_article( find_search_result( "article 2" ), { |
||||
"scenarios": [ "+Test Scenario 3 [TEST 3]", "+No scenario ID" ] |
||||
} ) |
||||
_check_scenarios( flask_app, all_scenarios, [ |
||||
[ "Test Scenario 1 [TEST 1]" ], |
||||
[ "Test Scenario 3 [TEST 3]", "No scenario ID" ] |
||||
] ) |
||||
|
||||
# add/remove scenarios to article #2 |
||||
_edit_article( find_search_result( "article 2" ), { |
||||
"scenarios": [ "+Test Scenario 1 [TEST 1]", "-Test Scenario 3 [TEST 3]" ] |
||||
} ) |
||||
_check_scenarios( flask_app, all_scenarios, [ |
||||
[ "Test Scenario 1 [TEST 1]" ], |
||||
[ "No scenario ID", "Test Scenario 1 [TEST 1]" ] |
||||
] ) |
||||
|
||||
# add an unknown scenario to article #1 |
||||
_edit_article( find_search_result( "article 1" ), { |
||||
"scenarios": [ "+new scenario [NEW]" ] |
||||
} ) |
||||
_check_scenarios( flask_app, all_scenarios, [ |
||||
[ "Test Scenario 1 [TEST 1]", "new scenario [NEW]" ], |
||||
[ "No scenario ID", "Test Scenario 1 [TEST 1]" ] |
||||
] ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _check_scenarios( flask_app, all_scenarios, expected ): |
||||
"""Check the scenarios of the test articles.""" |
||||
|
||||
# update the complete list of scenarios |
||||
# NOTE: Unlike tags, scenarios remain in the database even if no-one is referencing them, |
||||
# so we need to track them over the life of the entire series of tests. |
||||
for scenarios in expected: |
||||
all_scenarios.update( scenarios ) |
||||
|
||||
# check the scenarios in the UI |
||||
for article_no,scenarios in enumerate( expected ): |
||||
|
||||
# check the scenarios for the next article |
||||
sr = find_search_result( "article {}".format( 1+article_no ) ) |
||||
find_child( ".edit", sr ).click() |
||||
dlg = wait_for_elem( 2, "#modal-form" ) |
||||
select = ReactSelect( find_child( ".scenarios .react-select", dlg ) ) |
||||
assert select.get_multiselect_values() == scenarios |
||||
|
||||
# check that the list of available scenarios is correct |
||||
assert select.get_multiselect_choices() == \ |
||||
sorted( all_scenarios.difference( scenarios ), key=lambda s: s.lower() ) |
||||
|
||||
# close the dialog |
||||
find_child( "button.cancel", dlg ).click() |
||||
|
||||
# check the scenarios in the database |
||||
url = flask_app.url_for( "get_scenarios" ) |
||||
scenarios = json.load( urllib.request.urlopen( url ) ) |
||||
assert set( _make_scenario_display_name(a) for a in scenarios.values() ) == all_scenarios |
||||
|
||||
def _make_scenario_display_name( scenario ): |
||||
"""Generate the display name for a scenario.""" |
||||
if scenario["scenario_display_id"]: |
||||
return "{} [{}]".format( scenario["scenario_name"], scenario["scenario_display_id"] ) |
||||
else: |
||||
return scenario["scenario_name"] |
||||
@ -0,0 +1,173 @@ |
||||
#!/usr/bin/env python3 |
||||
""" Import scenarios from ROAR into our database. |
||||
|
||||
Download this file somewhere: |
||||
https://vasl-templates.org/services/roar/scenario-index.json |
||||
Then pass it in to this program. |
||||
|
||||
This script should be run to initialize a new database with a list of known scenarios, but we also |
||||
have to consider the possibilty that it will be run on a database that has been in use for a while, |
||||
and has new scenarios that have been added by the user i.e. we need to try to match scenarios |
||||
that are already in the database with what's in ROAR. |
||||
""" |
||||
|
||||
import sys |
||||
import os |
||||
import json |
||||
|
||||
import sqlalchemy |
||||
from sqlalchemy import text |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def main(): |
||||
"""Import ROAR scenarios into our database.""" |
||||
|
||||
# parse the command line arguments |
||||
if len(sys.argv) != 3: |
||||
print( "Usage: {} <dbconn> <scenario-index>".format( os.path.split(__file__)[0] ) ) |
||||
print( " dbconn: database connection string e.g. \"sqlite:///~/asl-articles.db\"" ) |
||||
print( " scenario-index: the ROAR scenario index file." ) |
||||
sys.exit( 0 ) |
||||
dbconn = sys.argv[1] |
||||
roar_fname = sys.argv[2] |
||||
|
||||
# load the ROAR scenario data |
||||
stats, warnings = import_roar_scenarios( dbconn, roar_fname, progress=print ) |
||||
|
||||
# output any warnings |
||||
for warning in warnings: |
||||
print() |
||||
print( "\n".join( warning ) ) |
||||
|
||||
# output stats |
||||
row_count = lambda n: "1 row" if n == 1 else "{} rows".format(n) |
||||
print() |
||||
print( "New scenarios added: {}".format( row_count( stats["nInserts"] ) ) ) |
||||
print( "Scenarios updated: {}".format( row_count( stats["nUpdates"] ) ) ) |
||||
print( "Duplicates ignored: {}".format( row_count( stats["nDupes"] ) ) ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def import_roar_scenarios( dbconn, roar_data, progress=None ): |
||||
"""Import scenarios from ROAR into our database.""" |
||||
|
||||
# initialize |
||||
stats = { "nInserts": 0, "nUpdates": 0, "nDupes": 0 } |
||||
warnings = [] |
||||
def log_progress( msg, *args, **kwargs ): |
||||
if progress: |
||||
progress( msg.format( *args, **kwargs ) ) |
||||
|
||||
# load the ROAR scenarios |
||||
if isinstance( roar_data, str ): |
||||
log_progress( "Loading scenarios: {}", roar_data ) |
||||
roar_data = json.load( open( roar_data, "r" ) ) |
||||
else: |
||||
assert isinstance( roar_data, dict ) |
||||
log_progress( "- Last updated: {}".format( roar_data.get("_lastUpdated_","(unknown)") ) ) |
||||
scenarios = { k: v for k,v in roar_data.items() if k.isdigit() } |
||||
log_progress( "- Loaded {} scenarios.".format( len(scenarios) ) ) |
||||
|
||||
# update the database |
||||
# NOTE: We can never delete rows from the scenario table, since Article's reference them by ID. |
||||
engine = sqlalchemy.create_engine( dbconn ) |
||||
conn = engine.connect() |
||||
with conn.begin(): |
||||
|
||||
for roar_id,scenario in scenarios.items(): |
||||
|
||||
# prepare the next scenario |
||||
vals = { "roar_id": roar_id, "display_id": scenario["scenario_id"], "name": scenario["name"] } |
||||
|
||||
# check if we already have the scenario |
||||
row = find_existing_scenario( conn, roar_id, scenario ) |
||||
if row: |
||||
|
||||
# yup - check if the details are the same |
||||
if row["scenario_display_id"] == scenario["scenario_id"] and row["scenario_name"] == scenario["name"]: |
||||
# yup - nothing to do |
||||
stats[ "nDupes" ] += 1 |
||||
continue |
||||
|
||||
# nope - update the row |
||||
warnings.append( [ |
||||
"Data mismatch for ROAR ID {}:".format( roar_id ), |
||||
"- Old details: {}: {}".format( row["scenario_display_id"], row["scenario_name"] ), |
||||
"- Updating to: {}: {}".format( scenario["scenario_id"], scenario["name"] ) |
||||
] ) |
||||
conn.execute( text( "UPDATE scenario" |
||||
" SET scenario_display_id = :display_id, scenario_name = :name" |
||||
" WHERE scenario_id = :scenario_id" ), |
||||
scenario_id=row["scenario_id"], **vals |
||||
) |
||||
stats[ "nUpdates" ] += 1 |
||||
|
||||
else: |
||||
|
||||
# nope - insert a new row |
||||
conn.execute( text( "INSERT INTO scenario" |
||||
" ( scenario_roar_id, scenario_display_id, scenario_name )" |
||||
" VALUES ( :roar_id, :display_id, :name )" ), |
||||
**vals |
||||
) |
||||
stats[ "nInserts" ] += 1 |
||||
|
||||
conn.close() |
||||
|
||||
return stats, warnings |
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
||||
|
||||
def find_existing_scenario( conn, roar_id, scenario ): |
||||
"""Check if the scenario is already in the database. |
||||
|
||||
Identifying existing scenarios in the database is complicated by the fact that the user can add scenarios |
||||
from the webapp, and the only required field is the scenario name e.g. we have to handle things like: |
||||
- this script is run to load an empty database |
||||
- the user manually adds a new scenario, but only provides its name |
||||
- the scenario is added to ROAR (with a ROAR ID and scenario ID) |
||||
- this script is run again. |
||||
In this case, we want to match the two scenarios and update the existing row, not create a new one. |
||||
""" |
||||
|
||||
def find_match( sql, **args ): |
||||
rows = list( conn.execute( text(sql), **args ) ) |
||||
if rows: |
||||
assert len(rows) == 1 |
||||
return rows[0] |
||||
else: |
||||
return None |
||||
|
||||
# try to match by ROAR ID |
||||
row = find_match( "SELECT * FROM scenario WHERE scenario_roar_id = :roar_id", |
||||
roar_id = roar_id |
||||
) |
||||
if row: |
||||
return row |
||||
|
||||
# try to match by scenario display ID and name |
||||
row = find_match( "SELECT * FROM scenario" |
||||
" WHERE (scenario_roar_id IS NULL OR scenario_roar_id = '' )" |
||||
" AND scenario_display_id = :scenario_id AND scenario_name = :name", |
||||
scenario_id=scenario["scenario_id"], name=scenario["name"] |
||||
) |
||||
if row: |
||||
return row |
||||
|
||||
# try to match by scenario name |
||||
row = find_match( "SELECT * FROM scenario" |
||||
" WHERE (scenario_roar_id IS NULL OR scenario_roar_id = '' )" |
||||
" AND ( scenario_display_id IS NULL OR scenario_display_id = '' )" |
||||
" AND scenario_name = :name", |
||||
name=scenario["name"] |
||||
) |
||||
if row: |
||||
return row |
||||
|
||||
return None |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
Loading…
Reference in new issue