parent
189d72725c
commit
197a665b10
@ -0,0 +1,150 @@ |
||||
""" Generate the database report. """ |
||||
|
||||
import urllib.request |
||||
import urllib.error |
||||
import hashlib |
||||
from collections import defaultdict |
||||
|
||||
from flask import request, jsonify, abort |
||||
|
||||
from asl_articles import app, db |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/db-report/row-counts" ) |
||||
def get_db_row_counts(): |
||||
"""Get the database row counts.""" |
||||
results = {} |
||||
for table_name in [ |
||||
"publisher", "publication", "article", "author", |
||||
"publisher_image", "publication_image", "article_image", |
||||
"scenario" |
||||
]: |
||||
query = db.engine.execute( "SELECT count(*) FROM {}".format( table_name ) ) |
||||
results[ table_name ] = query.scalar() |
||||
return jsonify( results ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/db-report/links" ) |
||||
def get_db_links(): |
||||
"""Get all links in the database.""" |
||||
|
||||
# initialize |
||||
results = {} |
||||
|
||||
def find_db_links( table_name, col_names ): |
||||
links = [] |
||||
query = db.engine.execute( "SELECT * FROM {}".format( table_name ) ) |
||||
for row in query: |
||||
url = row[ col_names[1] ] |
||||
if not url: |
||||
continue |
||||
obj_id = row[ col_names[0] ] |
||||
name = col_names[2]( row ) if callable( col_names[2] ) else row[ col_names[2] ] |
||||
links.append( [ obj_id, name, url ] ) |
||||
results[ table_name ] = links |
||||
|
||||
# find all links |
||||
find_db_links( "publisher", [ |
||||
"publ_id", "publ_url", "publ_name" |
||||
] ) |
||||
find_db_links( "publication", [ |
||||
"pub_id", "pub_url", _get_pub_name |
||||
] ) |
||||
find_db_links( "article", [ |
||||
"article_id", "article_url", "article_title" |
||||
] ) |
||||
|
||||
return jsonify( results ) |
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
||||
|
||||
@app.route( "/db-report/check-link", methods=["POST"] ) |
||||
def check_db_link(): |
||||
"""Check if a link appears to be working.""" |
||||
url = request.args.get( "url" ) |
||||
try: |
||||
resp = urllib.request.urlopen( |
||||
urllib.request.Request( url, method="HEAD" ) |
||||
) |
||||
except urllib.error.URLError as ex: |
||||
code = getattr( ex, "code", None ) |
||||
if code: |
||||
abort( code ) |
||||
abort( 400 ) |
||||
if resp.code != 200: |
||||
abort( resp.code ) |
||||
return "ok" |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
@app.route( "/db-report/images" ) |
||||
def get_db_images(): |
||||
"""Analyze the images stored in the database.""" |
||||
|
||||
# initialize |
||||
results = {} |
||||
image_hashes = defaultdict( list ) |
||||
|
||||
def find_images( table_name, col_names, get_name ): |
||||
|
||||
# find rows in the specified table that have images |
||||
sql = "SELECT {cols}, image_data" \ |
||||
" FROM {table}_image LEFT JOIN {table}" \ |
||||
" ON {table}_image.{id_col} = {table}.{id_col}".format( |
||||
cols = ",".join( "{}.{}".format( table_name, c ) for c in col_names ), |
||||
table = table_name, |
||||
id_col = col_names[0] |
||||
) |
||||
rows = [ |
||||
dict( row ) |
||||
for row in db.engine.execute( sql ) |
||||
] |
||||
|
||||
# save the image hashes |
||||
for row in rows: |
||||
image_hash = hashlib.md5( row["image_data"] ).hexdigest() |
||||
image_hashes[ image_hash ].append( [ |
||||
table_name, row[col_names[0]], get_name(row) |
||||
] ) |
||||
|
||||
# save the image sizes |
||||
image_sizes = [ |
||||
[ len(row["image_data"]), row[col_names[0]], get_name(row) ] |
||||
for row in rows |
||||
] |
||||
image_sizes.sort( key = lambda r: r[0], reverse=True ) |
||||
results[ table_name ] = image_sizes |
||||
|
||||
# look for images in each table |
||||
find_images( "publisher", |
||||
[ "publ_id", "publ_name" ], |
||||
lambda row: row["publ_name"] |
||||
) |
||||
find_images( "publication", |
||||
[ "pub_id", "pub_name", "pub_edition" ], |
||||
_get_pub_name |
||||
) |
||||
find_images( "article", |
||||
[ "article_id", "article_title" ], |
||||
lambda row: row["article_title"] |
||||
) |
||||
|
||||
# look for duplicate images |
||||
results["duplicates"] = {} |
||||
for image_hash, images in image_hashes.items(): |
||||
if len(images) == 1: |
||||
continue |
||||
results["duplicates"][ image_hash ] = images |
||||
|
||||
return results |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _get_pub_name( row ): |
||||
"""Get a publication's display name.""" |
||||
name = row["pub_name"] |
||||
if row["pub_edition"]: |
||||
name += " ({})".format( row["pub_edition"] ) |
||||
return name |
||||
@ -0,0 +1,42 @@ |
||||
{ |
||||
|
||||
"publisher": [ |
||||
{ "publ_id": 1, "publ_name": "Avalon Hill", "publ_url": "http://{FLASK}/ping" }, |
||||
{ "publ_id": 2, "publ_name": "Multiman Publishing", "publ_url": "http://{FLASK}/unknown" } |
||||
], |
||||
|
||||
"publication": [ |
||||
{ "pub_id": 10, "pub_name": "ASL Journal", "pub_edition": "1", "publ_id": 1, "pub_url": "/aslj-1.html" }, |
||||
{ "pub_id": 11, "pub_name": "ASL Journal", "pub_edition": "2", "publ_id": 1, "pub_url": "/aslj-2.html" }, |
||||
{ "pub_id": 20, "pub_name": "MMP News", "publ_id": 2 } |
||||
], |
||||
|
||||
"article": [ |
||||
{ "article_id": 100, "article_title": "ASLJ article 1", "pub_id": 10 }, |
||||
{ "article_id": 101, "article_title": "ASLJ article 2", "pub_id": 10 }, |
||||
{ "article_id": 110, "article_title": "ASLJ article 3", "pub_id": 11 }, |
||||
{ "article_id": 200, "article_title": "MMP article", "pub_id": 20, "article_url": "/mmp.html" }, |
||||
{ "article_id": 299, "article_title": "MMP publisher article", "publ_id": 2, "article_url": "/unknown" } |
||||
], |
||||
|
||||
"article_author": [ |
||||
{ "seq_no": 1, "article_id": 100, "author_id": 1000 }, |
||||
{ "seq_no": 2, "article_id": 100, "author_id": 1001 }, |
||||
{ "seq_no": 1, "article_id": 299, "author_id": 1000 } |
||||
], |
||||
"author": [ |
||||
{ "author_id": 1000, "author_name": "Joe Blow" }, |
||||
{ "author_id": 1001, "author_name": "Fred Nerk" }, |
||||
{ "author_id": 1999, "author_name": "Alan Smithee" } |
||||
], |
||||
|
||||
"article_scenario": [ |
||||
{ "seq_no": 1, "article_id": 100, "scenario_id": 2000 }, |
||||
{ "seq_no": 1, "article_id": 299, "scenario_id": 2001 } |
||||
], |
||||
"scenario": [ |
||||
{ "scenario_id": 2000, "scenario_display_id": "ASL 1", "scenario_name": "The Guards Counterattack" }, |
||||
{ "scenario_id": 2001, "scenario_display_id": "ASL 5", "scenario_name": "Hill 621" } |
||||
] |
||||
|
||||
} |
||||
@ -0,0 +1 @@ |
||||
ASL Journal #1. |
||||
@ -0,0 +1 @@ |
||||
ASL Journal #2. |
||||
@ -0,0 +1 @@ |
||||
Multiman Publishing. |
||||
@ -0,0 +1,219 @@ |
||||
""" Test the database reports. """ |
||||
|
||||
import os |
||||
import itertools |
||||
import re |
||||
|
||||
from asl_articles.search import SEARCH_ALL |
||||
from asl_articles.tests.test_publishers import edit_publisher |
||||
from asl_articles.tests.test_publications import edit_publication |
||||
from asl_articles.tests.test_articles import edit_article |
||||
from asl_articles.tests.utils import init_tests, \ |
||||
select_main_menu_option, select_sr_menu_option, check_ask_dialog, \ |
||||
do_search, find_search_result, \ |
||||
wait_for, wait_for_elem, find_child, find_children |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def test_db_report( webdriver, flask_app, dbconn ): |
||||
"""Test the database report.""" |
||||
|
||||
# initialize |
||||
init_tests( webdriver, flask_app, dbconn, fixtures="db-report.json" ) |
||||
|
||||
# check the initial report |
||||
row_counts, links, dupe_images, image_sizes = _get_db_report() |
||||
assert row_counts == { |
||||
"publishers": 2, "publications": 3, "articles": 5, |
||||
"authors": 3, "scenarios": 2 |
||||
} |
||||
assert links == { |
||||
"publishers": [ 2, [] ], |
||||
"publications": [ 2, [] ], |
||||
"articles": [ 2, [] ], |
||||
} |
||||
assert dupe_images == [] |
||||
assert image_sizes == {} |
||||
|
||||
# add some images |
||||
results = do_search( SEARCH_ALL ) |
||||
publ_sr = find_search_result( "Avalon Hill", results ) |
||||
fname = os.path.join( os.path.split(__file__)[0], "fixtures/images/1.gif" ) |
||||
edit_publisher( publ_sr, { "image": fname } ) |
||||
pub_sr = find_search_result( "ASL Journal (1)", results ) |
||||
fname = os.path.join( os.path.split(__file__)[0], "fixtures/images/2.gif" ) |
||||
edit_publication( pub_sr, { "image": fname } ) |
||||
article_sr = find_search_result( "ASLJ article 1", results ) |
||||
fname = os.path.join( os.path.split(__file__)[0], "fixtures/images/3.gif" ) |
||||
edit_article( article_sr, { "image": fname } ) |
||||
article_sr = find_search_result( "ASLJ article 2", results ) |
||||
fname = os.path.join( os.path.split(__file__)[0], "fixtures/images/3.gif" ) |
||||
edit_article( article_sr, { "image": fname } ) |
||||
|
||||
# check the updated report |
||||
row_counts, _, dupe_images, image_sizes = _get_db_report() |
||||
assert row_counts == { |
||||
"publishers": 2, "publisher_images": 1, |
||||
"publications": 3, "publication_images": 1, |
||||
"articles": 5, "article_images": 2, |
||||
"authors": 3, "scenarios": 2 |
||||
} |
||||
assert dupe_images == [ |
||||
[ "f0457ea742376e76ff276ce62c7a8540", "/images/article/100", |
||||
( "ASLJ article 1", "/article/100" ), |
||||
( "ASLJ article 2", "/article/101" ), |
||||
] |
||||
] |
||||
assert image_sizes == { |
||||
"publishers": [ |
||||
( "Avalon Hill", "/publisher/1", "/images/publisher/1" ), |
||||
], |
||||
"publications": [ |
||||
( "ASL Journal (1)", "/publication/10", "/images/publication/10" ), |
||||
], |
||||
"articles": [ |
||||
( "ASLJ article 1", "/article/100", "/images/article/100" ), |
||||
( "ASLJ article 2", "/article/101", "/images/article/101" ), |
||||
] |
||||
} |
||||
|
||||
# delete all the publishers (and associated objects), then check the updated report |
||||
results = do_search( SEARCH_ALL ) |
||||
publ_sr = find_search_result( "Avalon Hill", results ) |
||||
select_sr_menu_option( publ_sr, "delete" ) |
||||
check_ask_dialog( "Delete this publisher?", "ok" ) |
||||
publ_sr = find_search_result( "Multiman Publishing" ) |
||||
select_sr_menu_option( publ_sr, "delete" ) |
||||
check_ask_dialog( "Delete this publisher?", "ok" ) |
||||
row_counts, links, dupe_images, image_sizes = _get_db_report() |
||||
assert row_counts == { |
||||
"publishers": 0, "publications": 0, "articles": 0, |
||||
"authors": 3, "scenarios": 2 |
||||
} |
||||
assert links == { |
||||
"publishers": [ 0, [] ], |
||||
"publications": [ 0, [] ], |
||||
"articles": [ 0, [] ], |
||||
} |
||||
assert dupe_images == [] |
||||
assert image_sizes == {} |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def test_check_db_links( webdriver, flask_app, dbconn ): |
||||
"""Test checking links in the database.""" |
||||
|
||||
# initialize |
||||
init_tests( webdriver, flask_app, dbconn, docs="docs/", fixtures="db-report.json" ) |
||||
|
||||
# check the initial report |
||||
_, links, _, _ = _get_db_report() |
||||
assert links == { |
||||
"publishers": [ 2, [] ], |
||||
"publications": [ 2, [] ], |
||||
"articles": [ 2, [] ], |
||||
} |
||||
|
||||
# check the links |
||||
btn = find_child( "#db-report button.check-links" ) |
||||
btn.click() |
||||
status = find_child( "#db-report .db-links .status-msg" ) |
||||
wait_for( 10, lambda: status.text == "Checked 6 links." ) |
||||
|
||||
# check the updated report |
||||
_, links, _, _ = _get_db_report() |
||||
assert links == { |
||||
"publishers": [ 2, [ |
||||
( "Multiman Publishing", "/publisher/2", "HTTP 404: http://{FLASK}/unknown" ) |
||||
] ], |
||||
"publications": [ 2, [] ], |
||||
"articles": [ 2, [ |
||||
( "MMP publisher article", "/article/299", "HTTP 404: /unknown" ) |
||||
] ], |
||||
} |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _get_db_report(): #pylint: disable=too-many-locals |
||||
"""Generate the database report.""" |
||||
|
||||
# generate the report |
||||
select_main_menu_option( "db-report" ) |
||||
wait_for_elem( 2, "#db-report .db-images" ) |
||||
|
||||
# unload the row counts |
||||
row_counts = {} |
||||
table = find_child( "#db-report .db-row-counts" ) |
||||
for row in find_children( "tr", table ): |
||||
cells = find_children( "td", row ) |
||||
mo = re.search( r"^(\d+)( \((\d+) images?\))?$", cells[1].text ) |
||||
key = cells[0].text.lower()[:-1] |
||||
row_counts[ key ] = int( mo.group(1) ) |
||||
if mo.group( 3 ): |
||||
row_counts[ key[:-1] + "_images" ] = int( mo.group(3) ) |
||||
|
||||
# unload the links |
||||
links = {} |
||||
table = find_child( "#db-report .db-links" ) |
||||
last_key = None |
||||
for row in find_children( "tr", table ): |
||||
cells = find_children( "td", row ) |
||||
if len(cells) == 2: |
||||
last_key = cells[0].text.lower()[:-1] |
||||
links[ last_key ] = [ int( cells[1].text ) , [] ] |
||||
else: |
||||
mo = re.search( r"^(.+) \((.+)\)$", cells[0].text ) |
||||
tags = find_children( "a", cells[0] ) |
||||
url = _fixup_url( tags[0].get_attribute( "href" ) ) |
||||
links[ last_key ][1].append( ( mo.group(1), url, mo.group(2) ) ) |
||||
|
||||
# unload duplicate images |
||||
dupe_images = [] |
||||
for row in find_children( "#db-report .dupe-analysis .dupe-image" ): |
||||
elem = find_child( ".caption .hash", row ) |
||||
mo = re.search( r"^\(md5:(.+)\)$", elem.text ) |
||||
image_hash = mo.group(1) |
||||
image_url = _fixup_url( find_child( "img", row ).get_attribute( "src" ) ) |
||||
parents = [] |
||||
for entry in find_children( ".collapsible li", row ): |
||||
url = _fixup_url( find_child( "a", entry ).get_attribute( "href" ) ) |
||||
parents.append( ( entry.text, url ) ) |
||||
dupe_images.append( list( itertools.chain( |
||||
[ image_hash, image_url ], parents |
||||
) ) ) |
||||
|
||||
# unload the image sizes |
||||
tab_ctrl = find_child( "#db-report .db-images .react-tabs" ) |
||||
image_sizes = {} |
||||
for tab in find_children( ".react-tabs__tab", tab_ctrl ): |
||||
key = tab.text.lower() |
||||
tab_id = tab.get_attribute( "id" ) |
||||
tab.click() |
||||
sel = ".react-tabs__tab-panel[aria-labelledby='{}'].react-tabs__tab-panel--selected".format( tab_id ) |
||||
tab_page = wait_for( 2, |
||||
lambda: find_child( sel, tab_ctrl ) #pylint: disable=cell-var-from-loop |
||||
) |
||||
parents = [] |
||||
for row_no, row in enumerate( find_children( "table.image-sizes tr", tab_page ) ): |
||||
if row_no == 0: |
||||
continue |
||||
cells = find_children( "td", row ) |
||||
image_url = _fixup_url( find_child( "img", cells[0] ).get_attribute( "src" ) ) |
||||
url = _fixup_url( find_child( "a", cells[2] ).get_attribute( "href" ) ) |
||||
parents.append( ( cells[2].text, url, image_url ) ) |
||||
if parents: |
||||
image_sizes[ key ] = parents |
||||
else: |
||||
assert tab_page.text == "No images found." |
||||
|
||||
return row_counts, links, dupe_images, image_sizes |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def _fixup_url( url ): |
||||
"""Fixup a URL to make it independent of its server.""" |
||||
url = re.sub( r"^http://[^/]+", "", url ) |
||||
pos = url.find( "?" ) |
||||
if pos >= 0: |
||||
url = url[:pos] |
||||
return url |
||||
@ -1,78 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
""" Check the database for broken external document links. """ |
||||
|
||||
import sys |
||||
import os |
||||
import urllib.request |
||||
|
||||
import sqlalchemy |
||||
from sqlalchemy import text |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def main(): |
||||
"""Check the database for broken external document links.""" |
||||
|
||||
# parse the command line arguments |
||||
if len(sys.argv) != 3: |
||||
print( "Usage: {} <dbconn> <url-base>".format( os.path.split(__file__)[0] ) ) |
||||
print( " dbconn: database connection string e.g. \"sqlite:///~/asl-articles.db\"" ) |
||||
print( " url-base: Base URL for external documents e.g. http://localhost:3000/api/docs" ) |
||||
sys.exit( 0 ) |
||||
dbconn = sys.argv[1] |
||||
url_base = sys.argv[2] |
||||
|
||||
# connect to the database |
||||
engine = sqlalchemy.create_engine( dbconn ) |
||||
conn = engine.connect() |
||||
|
||||
def pub_name( row ): |
||||
name = row["pub_name"] |
||||
if row["pub_edition"]: |
||||
name += " ({})".format( row["pub_edition"] ) |
||||
return name |
||||
|
||||
# look for broken links |
||||
find_broken_links( conn, url_base, "publisher", [ |
||||
"publ_id", "publ_url", "publ_name" |
||||
] ) |
||||
find_broken_links( conn, url_base, "publication", [ |
||||
"pub_id", "pub_url", pub_name |
||||
] ) |
||||
find_broken_links( conn, url_base, "article", [ |
||||
"article_id", "article_url", "article_title" |
||||
] ) |
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - |
||||
|
||||
def find_broken_links( conn, url_base, table_name, col_names ): |
||||
"""Look for broken links.""" |
||||
|
||||
def check_url( url, row_id, name ): |
||||
|
||||
if not url.startswith( ( "http://", "https://" ) ): |
||||
url = os.path.join( url_base, url ) |
||||
url = url.replace( " ", "%20" ).replace( "#", "%23" ) |
||||
|
||||
#print( "Checking {}: {}".format( name, url ), file=sys.stderr ) |
||||
try: |
||||
buf = urllib.request.urlopen( url ).read() |
||||
except urllib.error.HTTPError: |
||||
buf = "" |
||||
if not buf: |
||||
print( "Broken link for \"{}\" (id={}): {}".format( name, row_id, url )) |
||||
|
||||
# check each row in the specified table |
||||
query = conn.execute( text( "SELECT * FROM {}".format( table_name ) ) ) |
||||
for row in query: |
||||
url = row[ col_names[1] ] |
||||
if not url: |
||||
continue |
||||
row_id = row[ col_names[0] ] |
||||
name = col_names[2]( row ) if callable( col_names[2] ) else row[ col_names[2] ] |
||||
check_url( url, row_id, name ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
@ -1,92 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
""" Geenrate a report on images in the database. """ |
||||
|
||||
import sys |
||||
import os |
||||
import hashlib |
||||
from collections import defaultdict |
||||
|
||||
import sqlalchemy |
||||
from sqlalchemy import text |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
def main(): |
||||
"""Report on images in the database.""" |
||||
|
||||
# parse the command line arguments |
||||
if len(sys.argv) != 2: |
||||
print( "Usage: {} <dbconn> <url-base>".format( os.path.split(__file__)[0] ) ) |
||||
print( " dbconn: database connection string e.g. \"sqlite:///~/asl-articles.db\"" ) |
||||
sys.exit( 0 ) |
||||
dbconn = sys.argv[1] |
||||
|
||||
# connect to the database |
||||
engine = sqlalchemy.create_engine( dbconn ) |
||||
conn = engine.connect() |
||||
|
||||
# initialize |
||||
image_hashes = defaultdict( list ) |
||||
|
||||
def find_images( conn, table_name, col_names, get_name ): |
||||
|
||||
# find rows in the specified table that have images |
||||
sql = "SELECT {cols}, image_data" \ |
||||
" FROM {table}_image LEFT JOIN {table}" \ |
||||
" ON {table}_image.{id_col} = {table}.{id_col}".format( |
||||
cols = ",".join( "{}.{}".format( table_name, c ) for c in col_names ), |
||||
table=table_name, id_col=col_names[0] |
||||
) |
||||
rows = [ dict(row) for row in conn.execute( text( sql ) ) ] |
||||
|
||||
# save the image hashes |
||||
for row in rows: |
||||
image_hash = hashlib.md5( row["image_data"] ).hexdigest() |
||||
name = get_name( row ) |
||||
image_hashes[ image_hash ].append( name ) |
||||
|
||||
# output the results |
||||
rows = [ |
||||
[ len(row["image_data"]), row[col_names[0]], get_name(row) ] |
||||
for row in rows |
||||
] |
||||
rows.sort( key = lambda r: r[0], reverse=True ) |
||||
print( "=== {}s ({}) ===".format( table_name, len(rows) ) ) |
||||
print() |
||||
print( "{:>6} {:>5}".format( "size", "ID" ) ) |
||||
for row in rows: |
||||
print( "{:-6.1f} | {:5} | {}".format( row[0]/1024, row[1], row[2] ) ) |
||||
print() |
||||
|
||||
def get_pub_name( row ): |
||||
name = row["pub_name"] |
||||
if row["pub_edition"]: |
||||
name += " ({})".format( row["pub_edition"] ) |
||||
return name |
||||
|
||||
# look for images in each table |
||||
find_images( conn, "publisher", |
||||
[ "publ_id", "publ_name" ], |
||||
lambda r: r["publ_name"] |
||||
) |
||||
find_images( conn, "publication", |
||||
[ "pub_id", "pub_name", "pub_edition" ], |
||||
get_pub_name |
||||
) |
||||
find_images( conn, "article", |
||||
[ "article_id", "article_title" ], |
||||
lambda r: r["article_title"] |
||||
) |
||||
|
||||
# report on any duplicate images |
||||
for image_hash,images in image_hashes.items(): |
||||
if len(images) == 1: |
||||
continue |
||||
print( "Found duplicate images ({}):".format( image_hash ) ) |
||||
for image in images: |
||||
print( "- {}".format( image ) ) |
||||
|
||||
# --------------------------------------------------------------------- |
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
|
After Width: | Height: | Size: 4.3 KiB |
|
After Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 7.7 KiB After Width: | Height: | Size: 7.7 KiB |
|
After Width: | Height: | Size: 584 B |
@ -0,0 +1,24 @@ |
||||
#db-report { |
||||
border: 1px solid #ccc ; border-radius: 8px ; |
||||
background: #fff ; padding: 0.5em ; |
||||
} |
||||
#db-report .section { margin-top: 1em ; } |
||||
#db-report .section:first-of-type { margin-top: 0 ; } |
||||
|
||||
#db-report h2 { border: 1px solid #ccc ; padding: 0.1em 0.2em ; background: #eee ; margin-bottom: 0.25em ; font-size: 125% ; } |
||||
#db-report h2 img.loading { height: 0.75em ; margin-left: 0.25em ; } |
||||
|
||||
#db-report .db-row-counts .images { font-size: 80% ; font-style: italic ; } |
||||
|
||||
#db-report .db-links .check-links-frame { display: inline-block ; position: absolute ; right: 1em ; text-align: center ; } |
||||
#db-report .db-links button.check-links { margin-bottom: 0.2em ; padding: 0.25em 0.5em ; } |
||||
#db-report .db-links .check-links-frame .status-msg { font-size: 60% ; font-style: italic ; } |
||||
#db-report .db-links .link-errors { font-size: 80% ; list-style-image: url("/images/link-error-bullet.png") ; } |
||||
#db-report .db-links .link-errors .status { font-family: monospace ; font-style: italic ; } |
||||
|
||||
#db-report .db-images .dupe-analysis .collapsible { margin-bottom: 0.5em ; } |
||||
#db-report .db-images .dupe-analysis .hash { font-family: monospace ; font-size: 80% ; font-style: italic ; } |
||||
#db-report .db-images .image-sizes th { text-align: left ; font-weight: normal ; font-style: italic ; } |
||||
#db-report .db-images .image-sizes img { height: 0.9em ; } |
||||
#db-report .db-images .react-tabs__tab-list { margin-bottom: 0 ; font-weight: bold ; } |
||||
#db-report .db-images .react-tabs__tab-panel { border: 1px solid #aaa ; border-top-width: 0 ; padding: 0.25em 0.5em ; } |
||||
@ -0,0 +1,387 @@ |
||||
import React from "react" ; |
||||
import { Link } from "react-router-dom" ; |
||||
import { Tabs, TabList, TabPanel, Tab } from 'react-tabs'; |
||||
import 'react-tabs/style/react-tabs.css'; |
||||
import "./DbReport.css" ; |
||||
import { PreviewableImage } from "./PreviewableImage" ; |
||||
import { gAppRef } from "./App.js" ; |
||||
import { makeCollapsibleList, pluralString, isLink } from "./utils.js" ; |
||||
|
||||
const axios = require( "axios" ) ; |
||||
|
||||
// --------------------------------------------------------------------
|
||||
|
||||
export class DbReport extends React.Component |
||||
{ |
||||
// render the component
|
||||
render() { |
||||
return ( <div id="db-report"> |
||||
<div className="section"> <DbRowCounts /> </div> |
||||
<div className="section"> <DbLinks /> </div> |
||||
<div className="section"> <DbImages /> </div> |
||||
</div> |
||||
) ; |
||||
} |
||||
} |
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
class DbRowCounts extends React.Component |
||||
{ |
||||
|
||||
constructor( props ) { |
||||
|
||||
// initialize
|
||||
super( props ) ; |
||||
this.state = { |
||||
dbRowCounts: null, |
||||
} ; |
||||
|
||||
// get the database row counts
|
||||
axios.get( |
||||
gAppRef.makeFlaskUrl( "/db-report/row-counts" ) |
||||
).then( resp => { |
||||
this.setState( { dbRowCounts: resp.data } ) ; |
||||
} ).catch( err => { |
||||
gAppRef.showErrorResponse( "Can't get the database row counts", err ) ; |
||||
} ) ; |
||||
|
||||
} |
||||
|
||||
render() { |
||||
|
||||
// initialize
|
||||
const dbRowCounts = this.state.dbRowCounts ; |
||||
|
||||
// render the table rows
|
||||
function makeRowCountRow( tableName ) { |
||||
const tableName2 = tableName[0].toUpperCase() + tableName.substring(1) ; |
||||
let nRows ; |
||||
if ( dbRowCounts ) { |
||||
nRows = dbRowCounts[ tableName ] ; |
||||
const nImages = dbRowCounts[ tableName+"_image" ] ; |
||||
if ( nImages > 0 ) |
||||
nRows = ( <span> |
||||
{nRows} <span className="images">({pluralString(nImages,"image")})</span> |
||||
</span> |
||||
) ; |
||||
} |
||||
return ( <tr key={tableName}> |
||||
<td style={{paddingRight:"0.5em",fontWeight:"bold"}}> {tableName2}s: </td> |
||||
<td> {nRows} </td> |
||||
</tr> |
||||
) ; |
||||
} |
||||
let tableRows = [ "publisher", "publication", "article", "author", "scenario" ].map( |
||||
(tableName) => makeRowCountRow( tableName ) |
||||
) ; |
||||
|
||||
// render the component
|
||||
return ( <div className="db-row-counts"> |
||||
<h2> Content { !dbRowCounts && <img src="/images/loading.gif" className="loading" alt="Loading..." /> } </h2> |
||||
<table><tbody>{tableRows}</tbody></table> |
||||
</div> |
||||
) ; |
||||
} |
||||
|
||||
} |
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
class DbLinks extends React.Component |
||||
{ |
||||
|
||||
constructor( props ) { |
||||
|
||||
// initialize
|
||||
super( props ) ; |
||||
this.state = { |
||||
dbLinks: null, |
||||
linksToCheck: null, currLinkToCheck: null, isFirstLinkCheck: true, |
||||
checkLinksInProgress: false, checkLinksStatusMsg: null, |
||||
linkErrors: {}, |
||||
} ; |
||||
|
||||
// initialize
|
||||
this._getLinksToCheck() ; |
||||
|
||||
} |
||||
|
||||
render() { |
||||
|
||||
// initialize
|
||||
const dbLinks = this.state.dbLinks ; |
||||
|
||||
// render the table rows
|
||||
let tableRows = [] ; |
||||
for ( let key of [ "publisher", "publication", "article" ] ) { |
||||
const nDbLinks = dbLinks && dbLinks[key] ? dbLinks[key].length : null ; |
||||
const key2 = key[0].toUpperCase() + key.substring(1) + "s" ; |
||||
tableRows.push( <tr key={key}> |
||||
<td style={{paddingRight:"0.5em",fontWeight:"bold"}}> {key2}: </td> |
||||
<td style={{width:"100%"}}> {nDbLinks} </td> |
||||
</tr> |
||||
) ; |
||||
if ( this.state.linkErrors[ key ] ) { |
||||
// NOTE: Showing all the errors at once (e.g. not as a collapsible list) will be unwieldy
|
||||
// if there are a lot of them, but this shouldn't happen often, and if it does, the user
|
||||
// is likely to stop the check, fix the problem, then try again.
|
||||
let rows = [] ; |
||||
for ( let linkError of this.state.linkErrors[ key ] ) { |
||||
const url = gAppRef.makeAppUrl( "/" + linkError[0][0] + "/" + linkError[0][1] ) ; |
||||
const targetUrl = linkError[0][3] ; |
||||
const target = isLink( targetUrl ) |
||||
? <a href={targetUrl}>{targetUrl}</a> |
||||
: targetUrl ; |
||||
let errorMsg = linkError[1] && linkError[1] + ": " ; |
||||
rows.push( <li key={linkError[0]}> |
||||
<Link to={url} dangerouslySetInnerHTML={{__html:linkError[0][2]}} /> |
||||
<span className="status"> ({errorMsg}{target}) </span> |
||||
</li> |
||||
) ; |
||||
} |
||||
tableRows.push( <tr key={key+"-errors"}> |
||||
<td colSpan="2"> |
||||
<ul className="link-errors"> {rows} </ul> |
||||
</td> |
||||
</tr> |
||||
) ; |
||||
} |
||||
} |
||||
|
||||
// render the component
|
||||
const nLinksToCheck = this.state.linksToCheck ? this.state.linksToCheck.length - this.state.currLinkToCheck : null ; |
||||
const imageUrl = this.state.checkLinksInProgress ? "/images/loading.gif" : "/images/icons/check-db-links.png" ; |
||||
return ( <div className="db-links"> |
||||
<h2> Links { !dbLinks && <img src="/images/loading.gif" className="loading" alt="Loading..." /> } </h2> |
||||
{ this.state.linksToCheck && this.state.linksToCheck.length > 0 && ( |
||||
<div className="check-links-frame"> |
||||
<button className="check-links" style={{display:"flex"}} onClick={() => this.checkDbLinks()} > |
||||
<img src={imageUrl} style={{height:"1.25em",marginRight:"0.5em"}} alt="Check database links." /> |
||||
{ this.state.checkLinksInProgress ? "Stop checking" : "Check links (" + nLinksToCheck + ")" } |
||||
</button> |
||||
<div className="status-msg"> {this.state.checkLinksStatusMsg} </div> |
||||
</div> |
||||
) } |
||||
<table className="db-links" style={{width:"100%"}}><tbody>{tableRows}</tbody></table> |
||||
</div> |
||||
) ; |
||||
} |
||||
|
||||
checkDbLinks() { |
||||
// start/stop checking links
|
||||
const inProgress = ! this.state.checkLinksInProgress ; |
||||
this.setState( { checkLinksInProgress: inProgress } ) ; |
||||
if ( inProgress ) |
||||
this._checkNextLink() ; |
||||
} |
||||
|
||||
_checkNextLink( force ) { |
||||
|
||||
// check if this is the start of a new run
|
||||
if ( this.state.currLinkToCheck === 0 && !force ) { |
||||
// yup - reset the UI
|
||||
this.setState( { linkErrors: {} } ) ; |
||||
// NOTE: If the user is checking the links *again*, it could be because some links were flagged
|
||||
// during the first run, they've fixed them up, and want to check everything again. In this case,
|
||||
// we need to re-fetch the links from the database.
|
||||
if ( ! this.state.isFirstLinkCheck ) { |
||||
this._getLinksToCheck( |
||||
() => { this._checkNextLink( true ) ; }, |
||||
() => { this.setState( { checkLinksInProgress: false } ) ; } |
||||
) ; |
||||
return ; |
||||
} |
||||
} |
||||
|
||||
// check if this is the end of a run
|
||||
if ( this.state.currLinkToCheck >= this.state.linksToCheck.length ) { |
||||
// yup - reset the UI
|
||||
this.setState( { |
||||
checkLinksStatusMsg: "Checked " + pluralString( this.state.linksToCheck.length, "link" ) + ".", |
||||
currLinkToCheck: 0, // nb: to allow the user to check again
|
||||
checkLinksInProgress: false, |
||||
isFirstLinkCheck: false, |
||||
} ) ; |
||||
return ; |
||||
} |
||||
|
||||
// get the next link to check
|
||||
const linkToCheck = this.state.linksToCheck[ this.state.currLinkToCheck ] ; |
||||
this.setState( { currLinkToCheck: this.state.currLinkToCheck + 1 } ) ; |
||||
|
||||
let continueCheckLinks = () => { |
||||
// update the UI
|
||||
this.setState( { checkLinksStatusMsg: |
||||
"Checked " + this.state.currLinkToCheck + " of " + pluralString( this.state.linksToCheck.length, "link" ) + "..." |
||||
} ) ; |
||||
// check the next link
|
||||
if ( this.state.checkLinksInProgress ) |
||||
this._checkNextLink() ; |
||||
} |
||||
|
||||
// check the next link
|
||||
let url = linkToCheck[3] ; |
||||
if ( url.substr( 0, 14 ) === "http://{FLASK}" ) |
||||
url = gAppRef.makeFlaskUrl( url.substr( 14 ) ) ; |
||||
// NOTE: Because of CORS, we have to proxy URL's that don't belong to us via the backend :-/
|
||||
let req = isLink( url ) |
||||
? axios.post( gAppRef.makeFlaskUrl( "/db-report/check-link", {url:url} ) ) |
||||
: axios.head( gAppRef.makeExternalDocUrl( url ) ) ; |
||||
req.then( resp => { |
||||
// the link worked - continue checking links
|
||||
continueCheckLinks() ; |
||||
} ).catch( err => { |
||||
// the link failed - record the error
|
||||
let newLinkErrors = this.state.linkErrors ; |
||||
if ( newLinkErrors[ linkToCheck[0] ] === undefined ) |
||||
newLinkErrors[ linkToCheck[0] ] = [] ; |
||||
const errorMsg = err.response ? "HTTP " + err.response.status : null ; |
||||
newLinkErrors[ linkToCheck[0] ].push( [ linkToCheck, errorMsg ] ) ; |
||||
this.setState( { linkErrors: newLinkErrors } ) ; |
||||
// continue checking links
|
||||
continueCheckLinks() ; |
||||
} ) ; |
||||
|
||||
} |
||||
|
||||
_getLinksToCheck( onOK, onError ) { |
||||
// get the links in the database
|
||||
axios.get( |
||||
gAppRef.makeFlaskUrl( "/db-report/links" ) |
||||
).then( resp => { |
||||
const dbLinks = resp.data ; |
||||
// flatten the links to a list
|
||||
let linksToCheck = [] ; |
||||
for ( let key of [ "publisher", "publication", "article" ] ) { |
||||
for ( let row of dbLinks[key] ) { |
||||
linksToCheck.push( [ |
||||
key, row[0], row[1], row[2] |
||||
] ) ; |
||||
} |
||||
} |
||||
this.setState( { |
||||
dbLinks: resp.data, |
||||
linksToCheck: linksToCheck, |
||||
currLinkToCheck: 0, |
||||
} ) ; |
||||
if ( onOK ) |
||||
onOK() ; |
||||
} ).catch( err => { |
||||
gAppRef.showErrorResponse( "Can't get the database links", err ) ; |
||||
if ( onError ) |
||||
onError() ; |
||||
} ) ; |
||||
} |
||||
|
||||
} |
||||
|
||||
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
class DbImages extends React.Component |
||||
{ |
||||
|
||||
constructor( props ) { |
||||
|
||||
// initialize
|
||||
super( props ) ; |
||||
this.state = { |
||||
dbImages: null, |
||||
} ; |
||||
|
||||
// get the database images
|
||||
axios.get( |
||||
gAppRef.makeFlaskUrl( "/db-report/images" ) |
||||
).then( resp => { |
||||
this.setState( { dbImages: resp.data } ) ; |
||||
} ).catch( err => { |
||||
gAppRef.showErrorResponse( "Can't get the database images", err ) ; |
||||
} ) ; |
||||
|
||||
} |
||||
|
||||
render() { |
||||
|
||||
// initialize
|
||||
const dbImages = this.state.dbImages ; |
||||
|
||||
// render any duplicate images
|
||||
let dupeImages = [] ; |
||||
if ( dbImages ) { |
||||
for ( let hash in dbImages.duplicates ) { |
||||
let parents = [] ; |
||||
for ( let row of dbImages.duplicates[hash] ) { |
||||
const url = gAppRef.makeAppUrl( "/" + row[0] + "/" + row[1] ) ; |
||||
parents.push( |
||||
<Link to={url} dangerouslySetInnerHTML={{__html:row[2]}} /> |
||||
) ; |
||||
} |
||||
// NOTE: We just use the first row's image since, presumably, they will all be the same.
|
||||
const row = dbImages.duplicates[hash][ 0 ] ; |
||||
const imageUrl = gAppRef.makeFlaskImageUrl( row[0], row[1] ) ; |
||||
const caption = ( <span> |
||||
Found a duplicate image <span className="hash">(md5:{hash})</span> |
||||
</span> |
||||
) ; |
||||
dupeImages.push( <div className="dupe-image" style={{display:"flex"}} key={hash} > |
||||
<PreviewableImage url={imageUrl} style={{width:"3em",marginTop:"0.1em",marginRight:"0.5em"}} /> |
||||
{ makeCollapsibleList( caption, parents, 5, {flexGrow:1}, hash ) } |
||||
</div> |
||||
) ; |
||||
} |
||||
} |
||||
|
||||
// render the image sizes
|
||||
let tabList = [] ; |
||||
let tabPanels = [] ; |
||||
if ( dbImages ) { |
||||
function toKB( n ) { return ( n / 1024 ).toFixed( 1 ) ; } |
||||
for ( let key of [ "publisher", "publication", "article" ] ) { |
||||
const tableName2 = key[0].toUpperCase() + key.substring(1) ; |
||||
tabList.push( |
||||
<Tab key={key}> {tableName2+"s"} </Tab> |
||||
) ; |
||||
let rows = [] ; |
||||
for ( let row of dbImages[key] ) { |
||||
const url = gAppRef.makeAppUrl( "/" + key + "/" + row[1] ) ; |
||||
// NOTE: Loading every image will be expensive, but we assume we're talking to a local server.
|
||||
// Otherwise, we could use a generic "preview" image, and expand it out to the real image
|
||||
// when the user clicks on it.
|
||||
const imageUrl = gAppRef.makeFlaskImageUrl( key, row[1] ) ; |
||||
rows.push( <tr key={row}> |
||||
<td> <PreviewableImage url={imageUrl} /> </td> |
||||
<td> {toKB(row[0])} </td> |
||||
<td> <Link to={url} dangerouslySetInnerHTML={{__html:row[2]}} /> </td> |
||||
</tr> |
||||
) ; |
||||
} |
||||
tabPanels.push( <TabPanel key={key}> |
||||
{ rows.length === 0 ? "No images found." : |
||||
<table className="image-sizes"><tbody> |
||||
<tr><th style={{width:"1.25em"}}/><th style={{paddingRight:"0.5em"}}> Size (KB) </th><th> {tableName2} </th></tr> |
||||
{rows} |
||||
</tbody></table> |
||||
} |
||||
</TabPanel> |
||||
) ; |
||||
} |
||||
} |
||||
const imageSizes = tabList.length > 0 && ( <Tabs> |
||||
<TabList> {tabList} </TabList> |
||||
{tabPanels} |
||||
</Tabs> |
||||
) ; |
||||
|
||||
// render the component
|
||||
return ( <div className="db-images"> |
||||
<h2> Images { !dbImages && <img src="/images/loading.gif" className="loading" alt="Loading..." /> } </h2> |
||||
{ dupeImages.length > 0 && |
||||
<div className="dupe-analysis"> {dupeImages} </div> |
||||
} |
||||
{imageSizes} |
||||
</div> |
||||
) ; |
||||
|
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue