From 03ef3153f72da6d5e041b7de8b2a57c837e624c8 Mon Sep 17 00:00:00 2001 From: Taka Date: Fri, 30 Apr 2021 00:45:11 +1000 Subject: [PATCH] Updated the Docker environment. --- Dockerfile | 37 +++++- asl_rulebook2/webapp/__init__.py | 15 ++- asl_rulebook2/webapp/asop.py | 10 +- .../webapp/config/logging.yaml.example | 42 ++++++ docker/config/debug.cfg | 1 + run-container.sh | 125 ++++++++++++++++-- 6 files changed, 208 insertions(+), 22 deletions(-) create mode 100644 asl_rulebook2/webapp/config/logging.yaml.example create mode 100644 docker/config/debug.cfg diff --git a/Dockerfile b/Dockerfile index dd1282a..81c87d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,40 @@ # NOTE: Use the run-container.sh script to build and launch this container. -FROM centos:8 +FROM centos:8 AS base # update packages RUN dnf -y upgrade-minimal +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +FROM base AS build + # install Python -RUN dnf install -y python38 python3-pip && \ - pip3 install -U pip setuptools +# NOTE: The version of Python we want is newer than what's in Centos 8, +# so we have to install from source :-/ +RUN dnf -y groupinstall "Development Tools" && \ + dnf -y install openssl-devel bzip2-devel libffi-devel sqlite-devel +RUN cd /tmp && \ + dnf -y install wget && \ + wget https://www.python.org/ftp/python/3.8.7/Python-3.8.7.tgz && \ + tar xvf Python-3.8.7.tgz && \ + cd Python-3.8.7/ && \ + ./configure --enable-optimizations && \ + make install + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +FROM base + +# copy the Python installation from the build image +COPY --from=build /usr/local/bin/python3.8 /usr/local/bin/python3.8 +COPY --from=build /usr/local/lib/python3.8 /usr/local/lib/python3.8 +COPY --from=build /usr/local/bin/pip3 /usr/local/bin/pip3 +RUN ln -s /usr/local/bin/python3.8 /usr/local/bin/python3 + +# install requirements +RUN dnf -y install ghostscript && \ + dnf clean all # install the application requirements COPY requirements.txt requirements-dev.txt ./ @@ -17,9 +44,6 @@ RUN if [ -n "$CONTROL_TESTS_PORT" ]; then \ pip3 install -r requirements-dev.txt \ ; fi -# clean up -RUN dnf clean all - # install the application WORKDIR /app COPY asl_rulebook2/ ./asl_rulebook2/ @@ -28,6 +52,7 @@ RUN pip3 install --editable . # install the config files COPY docker/config/ ./asl_rulebook2/webapp/config/ +COPY asl_rulebook2/webapp/config/logging.yaml.example ./asl_rulebook2/webapp/config/logging.yaml # create a new user RUN useradd --create-home app diff --git a/asl_rulebook2/webapp/__init__.py b/asl_rulebook2/webapp/__init__.py index 09054ed..4a35dac 100644 --- a/asl_rulebook2/webapp/__init__.py +++ b/asl_rulebook2/webapp/__init__.py @@ -27,6 +27,12 @@ def _load_config( fname, section ): config_parser.read( fname ) app.config.update( dict( config_parser.items( section ) ) ) +def _set_config_from_env( key ): + """Set an app config setting from an environment variable.""" + val = os.environ.get( "DOCKER_" + key ) + if val: + app.config[ key ] = val + # --------------------------------------------------------------------- def _on_sigint( signum, stack ): #pylint: disable=unused-argument @@ -54,9 +60,12 @@ app = Flask( __name__ ) _load_config( "app.cfg", "System" ) _load_config( "site.cfg", "Site Config" ) _load_config( "debug.cfg", "Debug" ) -for key, val in app.config.items(): - if str( val ).isdigit(): - app.config[ key ] = int( val ) +for _key, _val in app.config.items(): + if str( _val ).isdigit(): + app.config[ _key ] = int( _val ) + +# load any config from environment variables (e.g. set in the Docker container) +_set_config_from_env( "DATA_DIR" ) # initialize logging _fname = os.path.join( CONFIG_DIR, "logging.yaml" ) diff --git a/asl_rulebook2/webapp/asop.py b/asl_rulebook2/webapp/asop.py index 4fb273e..b2bc607 100644 --- a/asl_rulebook2/webapp/asop.py +++ b/asl_rulebook2/webapp/asop.py @@ -28,16 +28,16 @@ def init_asop( startup_msgs, logger ): data_dir = app.config.get( "DATA_DIR" ) if not data_dir: return None, None, None - base_dir = os.path.join( data_dir, "asop/" ) - if not os.path.isdir( base_dir ): + dname = os.path.join( data_dir, "asop/" ) + if not os.path.isdir( dname ): return None, None, None - _asop_dir = base_dir - fname = os.path.join( base_dir, "asop.css" ) + _asop_dir = dname + fname = os.path.join( _asop_dir, "asop.css" ) if os.path.isfile( fname ): user_css_url = url_for( "get_asop_file", path="asop.css" ) # load the ASOP index - fname = os.path.join( base_dir, "index.json" ) + fname = os.path.join( _asop_dir, "index.json" ) _asop = load_data_file( fname, "ASOP index", False, logger, startup_msgs.error ) if not _asop: return None, None, None diff --git a/asl_rulebook2/webapp/config/logging.yaml.example b/asl_rulebook2/webapp/config/logging.yaml.example new file mode 100644 index 0000000..25ac6d7 --- /dev/null +++ b/asl_rulebook2/webapp/config/logging.yaml.example @@ -0,0 +1,42 @@ +version: 1 + +formatters: + standard: + format: "%(asctime)s | [%(name)s/%(levelname)s] %(message)s" + datefmt: "%H:%M:%S" + +handlers: + console: + class: "logging.StreamHandler" + formatter: "standard" + stream: "ext://sys.stdout" + file: + class: "logging.FileHandler" + formatter: "standard" + filename: "/tmp/asl-rulebook2.log" + mode: "w" + +root: + level: "WARNING" + handlers: [ "console", "file" ] +loggers: + werkzeug: + level: "WARNING" + handlers: [ "console", "file" ] + propagate: 0 + startup: + level: "WARNING" + handlers: [ "console", "file" ] + propagate: 0 + search: + level: "WARNING" + handlers: [ "console", "file" ] + propagate: 0 + prepare: + level: "WARNING" + handlers: [ "console", "file" ] + propagate: 0 + control_tests: + level: "WARNING" + handlers: [ "console", "file" ] + propagate: 0 diff --git a/docker/config/debug.cfg b/docker/config/debug.cfg new file mode 100644 index 0000000..b5f1c17 --- /dev/null +++ b/docker/config/debug.cfg @@ -0,0 +1 @@ +[Debug] diff --git a/run-container.sh b/run-container.sh index 63221c5..bce55a4 100755 --- a/run-container.sh +++ b/run-container.sh @@ -6,8 +6,13 @@ function main { # initialize - cd $( dirname "$0" ) + cd "$( dirname "$0" )" PORT=5020 + DATA_DIR= + QA_DIR= + ERRATA_DIR= + USER_ANNO_FILE= + ASOP_DIR= IMAGE_TAG=latest CONTAINER_NAME=asl-rulebook2 DETACH= @@ -19,7 +24,7 @@ function main print_help exit 0 fi - params="$(getopt -o p:t:d -l port:,tag:,name:,detach,no-build,control-tests-port:,help --name "$0" -- "$@")" + params="$(getopt -o p:d:t: -l port:,data:,qa:,errata:,annotations:,asop:,tag:,name:,detach,no-build,control-tests-port:,help --name "$0" -- "$@")" if [ $? -ne 0 ]; then exit 1; fi eval set -- "$params" while true; do @@ -27,13 +32,28 @@ function main -p | --port ) PORT=$2 shift 2 ;; + -d | --data ) + DATA_DIR=$2 + shift 2 ;; + --qa ) + QA_DIR=$2 + shift 2 ;; + --errata ) + ERRATA_DIR=$2 + shift 2 ;; + --annotations ) + USER_ANNO_FILE=$2 + shift 2 ;; + --asop ) + ASOP_DIR=$2 + shift 2 ;; -t | --tag ) IMAGE_TAG=$2 shift 2 ;; --name ) CONTAINER_NAME=$2 shift 2 ;; - -d | --detach ) + --detach ) DETACH=--detach shift 1 ;; --no-build ) @@ -52,6 +72,62 @@ function main esac done + # check the data directory + if [ -n "$DATA_DIR" ]; then + target=$( get_target DIR "$DATA_DIR" ) + if [ -z "$target" ]; then + echo "Can't find the data directory: $DATA_DIR" + exit 2 + fi + mpoint=/data/ + DATA_DIR_VOLUME="--volume $target:$mpoint" + DATA_DIR_ENV="--env DOCKER_DATA_DIR=$mpoint" + fi + + # check the Q+A directory + if [ -n "$QA_DIR" ]; then + target=$( get_target DIR "$QA_DIR" ) + if [ -z "$target" ]; then + echo "Can't find the Q+A directory: $QA_DIR" + exit 2 + fi + mpoint=/data/q+a/ + QA_DIR_VOLUME="--volume $target:$mpoint" + fi + + # check the errata directory + if [ -n "$ERRATA_DIR" ]; then + target=$( get_target DIR "$ERRATA_DIR" ) + if [ -z "$target" ]; then + echo "Can't find the errata directory: $ERRATA_DIR" + exit 2 + fi + mpoint=/data/errata/ + ERRATA_DIR_VOLUME="--volume $target:$mpoint" + fi + + # check the user annotations file + if [ -n "$USER_ANNO_FILE" ]; then + target=$( get_target FILE "$USER_ANNO_FILE" ) + if [ -z "$target" ]; then + echo "Can't find the user annotations: $USER_ANNO_FILE" + exit 2 + fi + mpoint=/data/annotations.json + USER_ANNO_VOLUME="--volume $target:$mpoint" + fi + + # check the ASOP directory + if [ -n "$ASOP_DIR" ]; then + target=$( get_target DIR "$ASOP_DIR" ) + if [ -z "$target" ]; then + echo "Can't find the ASOP directory: $ASOP_DIR" + exit 2 + fi + mpoint=/data/asop/ + ASOP_DIR_VOLUME="--volume $target:$mpoint" + fi + # check if testing has been enabled if [ -n "$CONTROL_TESTS_PORT" ]; then CONTROL_TESTS_PORT_BUILD="--build-arg CONTROL_TESTS_PORT=$CONTROL_TESTS_PORT" @@ -77,6 +153,12 @@ function main --publish $PORT:5020 \ -it --rm \ $CONTROL_TESTS_PORT_RUN \ + $DATA_DIR_VOLUME $DATA_DIR_ENV \ + $QA_DIR_VOLUME \ + $ERRATA_DIR_VOLUME \ + $ASOP_DIR_VOLUME \ + $USER_ANNO_VOLUME \ + $DETACH \ asl-rulebook2:$IMAGE_TAG \ 2>&1 \ | sed -e 's/^/ /' @@ -85,17 +167,44 @@ function main # --------------------------------------------------------------------- +function get_target { + local type=$1 + local target=$2 + + # check that the target exists + if [ "$type" == "FILE" ]; then + test -f "$target" || return + elif [ "$type" == "DIR" ]; then + test -d "$target" || return + elif [ "$type" == "FILE-OR-DIR" ]; then + ls "$target" >/dev/null 2>&1 || return + fi + + # convert the target to a full path + # FUDGE! I couldn't get the "docker run" command to work with spaces in the volume targets (although + # copying the generated command into the terminal worked fine) (and no, using ${var@Q} didn't help). + # So, the next best thing is to allow users to create symlinks to the targets :-/ + echo $( realpath --no-symlinks "$target" ) +} + +# --------------------------------------------------------------------- + function print_help { echo "`basename "$0"` {options}" cat <