firethorn

changeset 4251:e9d24ed64319 version-2.1.33

Merging [2.1.32-zrq-thread-pools] into [2.1.33]
author Dave Morris <dmr@roe.ac.uk>
date Fri Jun 07 04:40:50 2019 +0100 (2019-06-07)
parents 0464b6306f73 5be5fa0abfb8
children e59ced69f8e5
files docker/apache/Dockerfile docker/builder/Dockerfile docker/compose/client/Dockerfile docker/compose/tests/distictella/Dockerfile docker/compose/tests/distictella/firethorn-py/Dockerfile docker/fedora/Dockerfile docker/java/Dockerfile docker/python/3.4.2/Dockerfile docker/pythonlibs/Dockerfile docker/sql-proxy/Dockerfile docker/sql-tunnel/Dockerfile docker/ssh-client/Dockerfile docker/tester/Dockerfile docker/tomcat/Dockerfile firethorn-admin/pom.xml firethorn-admin/src/main/docker/firethorn/Dockerfile firethorn-annotations/pom.xml firethorn-cds/adql/pom.xml firethorn-cds/pom.xml firethorn-core/pom.xml firethorn-jdbc/pom.xml firethorn-logging/pom.xml firethorn-ogsadai/activity/client/pom.xml firethorn-ogsadai/activity/common/pom.xml firethorn-ogsadai/activity/context/pom.xml firethorn-ogsadai/activity/pom.xml firethorn-ogsadai/activity/server/pom.xml firethorn-ogsadai/astro/client/pom.xml firethorn-ogsadai/astro/pom.xml firethorn-ogsadai/astro/server/pom.xml firethorn-ogsadai/depends/pom.xml firethorn-ogsadai/distributed/client/pom.xml firethorn-ogsadai/distributed/common/pom.xml firethorn-ogsadai/distributed/pom.xml firethorn-ogsadai/distributed/server/pom.xml firethorn-ogsadai/firethorn/client/pom.xml firethorn-ogsadai/firethorn/common/pom.xml firethorn-ogsadai/firethorn/pom.xml firethorn-ogsadai/firethorn/server/pom.xml firethorn-ogsadai/metadata/client/pom.xml firethorn-ogsadai/metadata/pom.xml firethorn-ogsadai/pom.xml firethorn-ogsadai/webapp/pom.xml firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile firethorn-spring/pom.xml firethorn-stil/pom.xml firethorn-testing/pom.xml firethorn-webapp/pom.xml firethorn-webapp/src/main/docker/firethorn/Dockerfile integration/005/pom.xml integration/tester/Dockerfile pom.xml
line diff
     1.1 --- a/doc/notes/zrq/20190601-01-hg-merge.txt	Mon Jun 03 04:46:00 2019 +0100
     1.2 +++ b/doc/notes/zrq/20190601-01-hg-merge.txt	Fri Jun 07 04:40:50 2019 +0100
     1.3 @@ -99,6 +99,7 @@
     1.4      branch=2.1.24-zrq-userdata-drop
     1.5      branch=2.1.25-zrq-tap-controller
     1.6      branch=2.1.31-stv-cleanup
     1.7 +    branch=2.1.32-zrq-thread-pools
     1.8  
     1.9  
    1.10  # -----------------------------------------------------
    1.11 @@ -342,7 +343,7 @@
    1.12      >   WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
    1.13      >   Configure a credential helper to remove this warning. See
    1.14      >   https://docs.docker.com/engine/reference/commandline/login/#credentials-store
    1.15 -    >   
    1.16 +    >
    1.17      >   Login Succeeded
    1.18      >   The push refers to repository [docker.io/firethorn/fedora]
    1.19      >   8bd87b3d77b4: Pushed
    1.20 @@ -493,10 +494,10 @@
    1.21      >   File list transfer time: 0.000 seconds
    1.22      >   Total bytes sent: 130.60M
    1.23      >   Total bytes received: 5.97K
    1.24 -    >   
    1.25 +    >
    1.26      >   sent 130.60M bytes  received 5.97K bytes  13.75M bytes/sec
    1.27      >   total size is 130.54M  speedup is 1.00
    1.28 -    >   
    1.29 +    >
    1.30      >   building file list ...
    1.31      >   95 files to consider
    1.32      >   uk/org/ogsadai/ogsadai-basic-client/4.2.2198/_remote.repositories
    1.33 @@ -517,10 +518,10 @@
    1.34      >   File list transfer time: 0.000 seconds
    1.35      >   Total bytes sent: 7.33K
    1.36      >   Total bytes received: 336
    1.37 -    >   
    1.38 +    >
    1.39      >   sent 7.33K bytes  received 336 bytes  3.07K bytes/sec
    1.40      >   total size is 29.41M  speedup is 3,836.53
    1.41 -    >   
    1.42 +    >
    1.43      >   building file list ...
    1.44      >   31 files to consider
    1.45      >   uk/ac/starlink/stil-table/3.3.1/_remote.repositories
    1.46 @@ -541,10 +542,10 @@
    1.47      >   File list transfer time: 0.000 seconds
    1.48      >   Total bytes sent: 4.14K
    1.49      >   Total bytes received: 236
    1.50 -    >   
    1.51 +    >
    1.52      >   sent 4.14K bytes  received 236 bytes  8.75K bytes/sec
    1.53      >   total size is 1.23M  speedup is 281.07
    1.54 -    >   
    1.55 +    >
    1.56      >   building file list ...
    1.57      >   3868 files to consider
    1.58      >   antlr/antlr/2.7.7/_remote.repositories
    1.59 @@ -567,7 +568,7 @@
    1.60      >   File list transfer time: 0.000 seconds
    1.61      >   Total bytes sent: 401.16K
    1.62      >   Total bytes received: 22.37K
    1.63 -    >   
    1.64 +    >
    1.65      >   sent 401.16K bytes  received 22.37K bytes  19.70K bytes/sec
    1.66      >   total size is 107.40M  speedup is 253.57
    1.67  
     2.1 --- a/doc/notes/zrq/20190601-02-merge-tests.txt	Mon Jun 03 04:46:00 2019 +0100
     2.2 +++ b/doc/notes/zrq/20190601-02-merge-tests.txt	Fri Jun 07 04:40:50 2019 +0100
     2.3 @@ -335,7 +335,6 @@
     2.4                  )
     2.5  
     2.6  
     2.7 -
     2.8  # -------- -------- -------- --------
     2.9  # Bug testing.
    2.10  
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/doc/notes/zrq/20190603-01-testing.txt	Fri Jun 07 04:40:50 2019 +0100
     3.3 @@ -0,0 +1,412 @@
     3.4 +#
     3.5 +# <meta:header>
     3.6 +#   <meta:licence>
     3.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     3.8 +#
     3.9 +#     This information is free software: you can redistribute it and/or modify
    3.10 +#     it under the terms of the GNU General Public License as published by
    3.11 +#     the Free Software Foundation, either version 3 of the License, or
    3.12 +#     (at your option) any later version.
    3.13 +#
    3.14 +#     This information is distributed in the hope that it will be useful,
    3.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    3.17 +#     GNU General Public License for more details.
    3.18 +#
    3.19 +#     You should have received a copy of the GNU General Public License
    3.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    3.21 +#   </meta:licence>
    3.22 +# </meta:header>
    3.23 +#
    3.24 +#
    3.25 +
    3.26 +
    3.27 +# -----------------------------------------------------
    3.28 +# Create a new VM.
    3.29 +#[user@trop01]
    3.30 +
    3.31 +    createvm
    3.32 +
    3.33 +    >   INFO : Node name [Astoalith]
    3.34 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
    3.35 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
    3.36 +    >   INFO : Disc name [Astoalith.qcow]
    3.37 +    >   INFO : Disc size [16GiB]
    3.38 +
    3.39 +    vmname=Astoalith
    3.40 +
    3.41 +
    3.42 +# -----------------------------------------------------
    3.43 +# Define a hosts lookup function.
    3.44 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
    3.45 +#[user@trop01]
    3.46 +
    3.47 +    getipv4()
    3.48 +        {
    3.49 +        getent hosts "${1:?}" | cut -d ' ' -f 1
    3.50 +        }
    3.51 +
    3.52 +
    3.53 +#---------------------------------------------------------------------
    3.54 +# Update the ssh keys for the VM.
    3.55 +#[user@trop01]
    3.56 +
    3.57 +    # Remove the current key.
    3.58 +    ssh-keygen -q -R "${vmname:?}"
    3.59 +
    3.60 +    # Add the host key(s) to known_hosts
    3.61 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
    3.62 +
    3.63 +    # Add the IP address key(s) to known_hosts
    3.64 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
    3.65 +
    3.66 +
    3.67 +# -----------------------------------------------------
    3.68 +# Login to the virtual machine.
    3.69 +#[user@trop]
    3.70 +
    3.71 +    ssh "${vmname:?}"
    3.72 +
    3.73 +
    3.74 +# -----------------------------------------------------
    3.75 +# Install our secret function.
    3.76 +#[user@virtual]
    3.77 +
    3.78 +    mkdir "${HOME:?}/bin"
    3.79 +    cat > "${HOME:?}/bin/secret" << 'EOF'
    3.80 +#!/bin/sh
    3.81 +ssh -n \
    3.82 +    'dave@shepseskaf.roe.ac.uk' \
    3.83 +    "bin/secret '${1}'"
    3.84 +EOF
    3.85 +
    3.86 +    chmod u+x "${HOME:?}/bin/secret"
    3.87 +    secret 'frog'
    3.88 +
    3.89 +
    3.90 +# -----------------------------------------------------
    3.91 +# Create our chain properties.
    3.92 +#[user@virtual]
    3.93 +
    3.94 +    cat > "${HOME:?}/chain.properties" << EOF
    3.95 +
    3.96 +        buildtag=latest
    3.97 +
    3.98 +        metadata=$(pwgen 20 1)
    3.99 +        metauser=$(pwgen 20 1)
   3.100 +        metapass=$(pwgen 20 1)
   3.101 +
   3.102 +        usertype=mssql
   3.103 +        userhost=$(secret 'firethorn.user.host')
   3.104 +        userdata=$(secret 'firethorn.user.data')
   3.105 +        useruser=$(secret 'firethorn.user.user')
   3.106 +        userpass=$(secret 'firethorn.user.pass')
   3.107 +
   3.108 +        datatype=mssql
   3.109 +        datahost=$(secret 'firethorn.data.host')
   3.110 +        datadata=$(secret 'firethorn.data.data')
   3.111 +        datauser=$(secret 'firethorn.data.user')
   3.112 +        datapass=$(secret 'firethorn.data.pass')
   3.113 +
   3.114 +        tunneluser=$(secret 'ssh.tunnel.user')
   3.115 +        tunnelhost=$(secret 'ssh.tunnel.host')
   3.116 +
   3.117 +        admingroup=Hyaenidae
   3.118 +        adminuser=Aardwolf
   3.119 +        adminpass=$(pwgen 20 1)
   3.120 +
   3.121 +        guestgroup=Afrotheria
   3.122 +        guestuser=Hyrax
   3.123 +        guestpass=$(pwgen 20 1)
   3.124 +
   3.125 +        tapresource=Wilhelmina
   3.126 +        tapschemadata=data-$(pwgen 10 1)
   3.127 +        tapschemauser=user-$(pwgen 10 1)
   3.128 +        tapschemapass=pass-$(pwgen 10 1)
   3.129 +
   3.130 +EOF
   3.131 +
   3.132 +# -----------------------------------------------------
   3.133 +# Link our compose config.
   3.134 +#[user@virtual]
   3.135 +
   3.136 +    ln -sf "${HOME:?}/chain.properties" "${HOME:?}/.env"
   3.137 +
   3.138 +# -----------------------------------------------------
   3.139 +# Identify our location.
   3.140 +#[user@virtual]
   3.141 +
   3.142 +# Choose one.
   3.143 +# 'local' if VM is inside UoE
   3.144 +# 'remote' if VM is at outside UoE
   3.145 +# TODO Change these to 'internal' and 'external'
   3.146 +
   3.147 +    external=$(curl -4 --silent 'http://icanhazip.com/')
   3.148 +
   3.149 +    EDINBURGH='129.215.*'
   3.150 +
   3.151 +    if [[ ${external:?} == ${EDINBURGH} ]]
   3.152 +    then
   3.153 +        echo "Address [${external:?}] is in Edinburgh"
   3.154 +        #location=internal
   3.155 +        location=local
   3.156 +    else
   3.157 +        echo "Address [${external:?}] is not in Edinburgh"
   3.158 +        #location=external
   3.159 +        location=remote
   3.160 +    fi
   3.161 +
   3.162 +# -----------------------------------------------------
   3.163 +# Download our compose file.
   3.164 +#[user@virtual]
   3.165 +
   3.166 +    wget \
   3.167 +        --output-document "baryptera-${location:?}.yml" \
   3.168 +        "http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/tests/baryptera/baryptera-${location:?}.yml"
   3.169 +
   3.170 +
   3.171 +# -----------------------------------------------------
   3.172 +# Start our tests ...
   3.173 +#[user@virtual]
   3.174 +
   3.175 +    docker-compose \
   3.176 +        --file "baryptera-${location:?}.yml" \
   3.177 +        run \
   3.178 +            angela
   3.179 +
   3.180 +# -----------------------------------------------------
   3.181 +# -----------------------------------------------------
   3.182 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   3.183 +#[user@virtual]
   3.184 +
   3.185 +    sudo -s
   3.186 +
   3.187 +    container=baryptera_gillian_1
   3.188 +    container=stevedore_gillian_1
   3.189 +
   3.190 +    pushd $(
   3.191 +        docker inspect \
   3.192 +            "${container:?}" \
   3.193 +      | jq -r '
   3.194 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   3.195 +            '
   3.196 +            )
   3.197 +
   3.198 +    tail -f firethorn-debug.log
   3.199 +
   3.200 +# -----------------------------------------------------
   3.201 +# -----------------------------------------------------
   3.202 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   3.203 +#[user@virtual]
   3.204 +
   3.205 +    sudo -s
   3.206 +
   3.207 +    container=baryptera_jarmila_1
   3.208 +    container=stevedore_jarmila_1
   3.209 +
   3.210 +    pushd $(
   3.211 +        docker inspect \
   3.212 +            "${container:?}" \
   3.213 +      | jq -r '
   3.214 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   3.215 +            '
   3.216 +            )
   3.217 +
   3.218 +    tail -f ogsadai.log
   3.219 +
   3.220 +# -----------------------------------------------------
   3.221 +# -----------------------------------------------------
   3.222 +# Run our Python tests ...
   3.223 +#[user@python]
   3.224 +
   3.225 +import os
   3.226 +import uuid
   3.227 +import time
   3.228 +import firethorn as ftpy
   3.229 +
   3.230 +#
   3.231 +# Create our firethorn client (using named param).
   3.232 +firethorn = ftpy.Firethorn(
   3.233 +    endpoint = os.environ.get(
   3.234 +        'endpoint'
   3.235 +        )
   3.236 +    )
   3.237 +
   3.238 +#
   3.239 +# Login as the admin account.
   3.240 +firethorn.login(
   3.241 +    os.environ.get('adminuser'),
   3.242 +    os.environ.get('adminpass'),
   3.243 +    os.environ.get('admingroup')
   3.244 +    )
   3.245 +
   3.246 +#
   3.247 +# Create a JdbcResource to connect to the ATLAS database.
   3.248 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   3.249 +    "ATLAS JDBC resource",
   3.250 +    os.environ.get('datadata'),
   3.251 +    '*',
   3.252 +    os.environ.get('datatype'),
   3.253 +    os.environ.get('datahost'),
   3.254 +    os.environ.get('datauser'),
   3.255 +    os.environ.get('datapass')
   3.256 +    )
   3.257 +print(
   3.258 +    atlas_jdbc
   3.259 +    )
   3.260 +
   3.261 +#
   3.262 +# Create an AdqlResource to represent the JdbcResource.
   3.263 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   3.264 +    "ATLAS ADQL resource"
   3.265 +    )
   3.266 +print(
   3.267 +    atlas_adql
   3.268 +    )
   3.269 +
   3.270 +#
   3.271 +# Import the target JdbcSchema into AdqlSchema.
   3.272 +schema_names = [
   3.273 +    "ATLASDR1"
   3.274 +    ]
   3.275 +
   3.276 +for schema_name in schema_names:
   3.277 +    print(schema_name)
   3.278 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   3.279 +        schema_name,
   3.280 +        "dbo"
   3.281 +        )
   3.282 +    if (None != jdbc_schema):
   3.283 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   3.284 +        adql_schema = atlas_adql.import_jdbc_schema(
   3.285 +            jdbc_schema,
   3.286 +            schema_name,
   3.287 +            metadoc=metadoc
   3.288 +            )
   3.289 +
   3.290 +#
   3.291 +# Admin user
   3.292 +# -------- -------- -------- --------
   3.293 +# Normal user
   3.294 +#
   3.295 +
   3.296 +#
   3.297 +# Login using a guest account.
   3.298 +firethorn.login(
   3.299 +    str(uuid.uuid4()),
   3.300 +    str(uuid.uuid4()),
   3.301 +    None
   3.302 +    )
   3.303 +
   3.304 +#
   3.305 +# Create a new workspace.
   3.306 +workspace = firethorn.firethorn_engine.create_adql_resource(
   3.307 +    "Query resource"
   3.308 +    )
   3.309 +
   3.310 +#
   3.311 +# Import the ATLAS schemas into our workspace
   3.312 +for schema in atlas_adql.select_schemas():
   3.313 +    workspace.import_adql_schema(
   3.314 +        schema
   3.315 +        )
   3.316 +
   3.317 +#
   3.318 +# Create and run a query.
   3.319 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   3.320 +query_obj = workspace.create_query(
   3.321 +    query_str,
   3.322 +    "COMPLETED",
   3.323 +    None,
   3.324 +    3000000
   3.325 +    )
   3.326 +print(
   3.327 +    query_obj
   3.328 +    )
   3.329 +print(
   3.330 +    query_obj.table()
   3.331 +    )
   3.332 +print(
   3.333 +    query_obj.table().count()
   3.334 +    )
   3.335 +
   3.336 +#
   3.337 +# Iterate the metadata tree
   3.338 +for schema in atlas_adql.select_schemas():
   3.339 +    for table in schema.select_tables():
   3.340 +        print(
   3.341 +            "table  [{}][{}]".format(
   3.342 +                schema.name(),
   3.343 +                table.name()
   3.344 +                )
   3.345 +            )
   3.346 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   3.347 +            schema.name(),
   3.348 +            table.name()
   3.349 +            )
   3.350 +        query_obj = workspace.create_query(
   3.351 +            query_str,
   3.352 +            "COMPLETED",
   3.353 +            None,
   3.354 +            3000000
   3.355 +            )
   3.356 +        py_table = query_obj.table().as_astropy()
   3.357 +        py_table.pprint()
   3.358 +
   3.359 +#
   3.360 +# Run multiple queries in parallel
   3.361 +from concurrent.futures import ThreadPoolExecutor
   3.362 +import concurrent.futures
   3.363 +from datetime import datetime
   3.364 +
   3.365 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   3.366 +
   3.367 +def do_query(workspace, query_str, limit, delay):
   3.368 +    before = datetime.now()
   3.369 +    query_obj = workspace.create_query(
   3.370 +        query_str,
   3.371 +        "COMPLETED",
   3.372 +        None,
   3.373 +        200000,
   3.374 +            {
   3.375 +            "adql.query.limit.rows"  : limit,
   3.376 +            "adql.query.delay.every" : delay
   3.377 +            }
   3.378 +        )
   3.379 +    after = datetime.now()
   3.380 +    return (
   3.381 +        (after - before),
   3.382 +        query_obj.json_object.get("results").get("count")
   3.383 +        )
   3.384 +
   3.385 +def do_queries(workspace, query_str, threads, delay):
   3.386 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   3.387 +        futures = {
   3.388 +            executor.submit(
   3.389 +                do_query,
   3.390 +                workspace,
   3.391 +                query_str,
   3.392 +                limit,
   3.393 +                delay
   3.394 +                ): limit for limit in range(threads, 0, -1)
   3.395 +            }
   3.396 +        for future in concurrent.futures.as_completed(futures):
   3.397 +            print(
   3.398 +                future.result()[0],
   3.399 +                ':',
   3.400 +                future.result()[1]
   3.401 +                )
   3.402 +
   3.403 +for loop in range(1, 10):
   3.404 +    for threads in range(1, 50):
   3.405 +        for delay in range(1000, -100, -100):
   3.406 +            print("---- ", loop, threads, delay)
   3.407 +            do_queries(
   3.408 +                workspace,
   3.409 +                query_str,
   3.410 +                threads,
   3.411 +                delay
   3.412 +                )
   3.413 +
   3.414 +
   3.415 +
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/doc/notes/zrq/20190603-02-updates.txt	Fri Jun 07 04:40:50 2019 +0100
     4.3 @@ -0,0 +1,147 @@
     4.4 +#
     4.5 +# <meta:header>
     4.6 +#   <meta:licence>
     4.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     4.8 +#
     4.9 +#     This information is free software: you can redistribute it and/or modify
    4.10 +#     it under the terms of the GNU General Public License as published by
    4.11 +#     the Free Software Foundation, either version 3 of the License, or
    4.12 +#     (at your option) any later version.
    4.13 +#
    4.14 +#     This information is distributed in the hope that it will be useful,
    4.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    4.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    4.17 +#     GNU General Public License for more details.
    4.18 +#
    4.19 +#     You should have received a copy of the GNU General Public License
    4.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    4.21 +#   </meta:licence>
    4.22 +# </meta:header>
    4.23 +#
    4.24 +#
    4.25 +
    4.26 +    #
    4.27 +    # Dev branch to experiment with Thread pool allocations.
    4.28 +    # It is possible for the system to lock up under concurrent tests.
    4.29 +    # Incomplete callbacks from OGSA-DAI stuck waiting in the queue means everything stalls.
    4.30 +    #
    4.31 +
    4.32 +# -----------------------------------------------------
    4.33 +# Create a new development branch.
    4.34 +#[user@desktop]
    4.35 +
    4.36 +    devname=zrq-thread-pools
    4.37 +
    4.38 +    source "${HOME:?}/firethorn.settings"
    4.39 +    gedit  "${FIRETHORN_CODE:?}/doc/notes/zrq/20180302-02-hg-branch.txt" &
    4.40 +
    4.41 +
    4.42 +#---------------------------------------------------------------------
    4.43 +# Update our base container.
    4.44 +#[user@desktop]
    4.45 +
    4.46 +    source "${HOME:?}/firethorn.settings"
    4.47 +    pushd "${FIRETHORN_CODE:?}"
    4.48 +
    4.49 +        vi docker/fedora/Dockerfile
    4.50 +
    4.51 +        -   # FROM fedora:28
    4.52 +        -   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
    4.53 +        -   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
    4.54 +
    4.55 +        +   # FROM latest (fedora:29)
    4.56 +        +   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
    4.57 +        +   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
    4.58 +
    4.59 +
    4.60 +        vi docker/java/Dockerfile
    4.61 +
    4.62 +        ~   ENV JAVA_VERSION=12.0.0.33
    4.63 +
    4.64 +
    4.65 +        vi docker/tomcat/Dockerfile
    4.66 +
    4.67 +        -   ENV TOMCAT_VERSION 9.0.10
    4.68 +        +   ENV TOMCAT_VERSION 9.0.20
    4.69 +
    4.70 +
    4.71 +        vi docker/tomcat/Dockerfile
    4.72 +
    4.73 +        -   FROM postgres:10
    4.74 +        +   FROM postgres:11
    4.75 +
    4.76 +
    4.77 +    popd
    4.78 +
    4.79 +
    4.80 +# -----------------------------------------------------
    4.81 +# Build our Docker images.
    4.82 +#[user@desktop]
    4.83 +
    4.84 +    source "${HOME:?}/firethorn.settings"
    4.85 +    pushd "${FIRETHORN_CODE:?}"
    4.86 +
    4.87 +        export buildtag=$(hg branch)
    4.88 +
    4.89 +        docker-compose \
    4.90 +            --file docker/compose/images.yml \
    4.91 +            build
    4.92 +
    4.93 +    popd
    4.94 +
    4.95 +
    4.96 +#---------------------------------------------------------------------
    4.97 +# Update our Java dependencies.
    4.98 +#[user@desktop]
    4.99 +
   4.100 +    source "${HOME:?}/firethorn.settings"
   4.101 +    pushd "${FIRETHORN_CODE:?}"
   4.102 +
   4.103 +        mvn versions:display-dependency-updates
   4.104 +
   4.105 +    popd
   4.106 +
   4.107 +
   4.108 +#---------------------------------------------------------------------
   4.109 +# Update our Maven dependencies.
   4.110 +#[user@desktop]
   4.111 +
   4.112 +    source "${HOME:?}/firethorn.settings"
   4.113 +    pushd "${FIRETHORN_CODE:?}"
   4.114 +
   4.115 +        mvn versions:display-plugin-updates
   4.116 +
   4.117 +    popd
   4.118 +
   4.119 +
   4.120 +#---------------------------------------------------------------------
   4.121 +# Compile our Java code.
   4.122 +#[user@desktop]
   4.123 +
   4.124 +    source "${HOME:?}/firethorn.settings"
   4.125 +    pushd "${FIRETHORN_CODE:?}"
   4.126 +
   4.127 +        mvn clean install eclipse:eclipse
   4.128 +
   4.129 +    popd
   4.130 +
   4.131 +# -----------------------------------------------------
   4.132 +# Build our Java containers.
   4.133 +#[user@desktop]
   4.134 +
   4.135 +    source "${HOME:?}/firethorn.settings"
   4.136 +    pushd "${FIRETHORN_CODE:?}"
   4.137 +
   4.138 +        export buildtag=$(hg branch)
   4.139 +        pushd firethorn-ogsadai/webapp
   4.140 +            mvn docker:package
   4.141 +        popd
   4.142 +
   4.143 +        export buildtag=$(hg branch)
   4.144 +        pushd firethorn-webapp
   4.145 +            mvn docker:package
   4.146 +        popd
   4.147 +
   4.148 +    popd
   4.149 +
   4.150 +
     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/doc/notes/zrq/20190604-01-test-build.txt	Fri Jun 07 04:40:50 2019 +0100
     5.3 @@ -0,0 +1,758 @@
     5.4 +#
     5.5 +# <meta:header>
     5.6 +#   <meta:licence>
     5.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     5.8 +#
     5.9 +#     This information is free software: you can redistribute it and/or modify
    5.10 +#     it under the terms of the GNU General Public License as published by
    5.11 +#     the Free Software Foundation, either version 3 of the License, or
    5.12 +#     (at your option) any later version.
    5.13 +#
    5.14 +#     This information is distributed in the hope that it will be useful,
    5.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    5.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    5.17 +#     GNU General Public License for more details.
    5.18 +#
    5.19 +#     You should have received a copy of the GNU General Public License
    5.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    5.21 +#   </meta:licence>
    5.22 +# </meta:header>
    5.23 +#
    5.24 +#
    5.25 +
    5.26 +# -----------------------------------------------------
    5.27 +# Create a new VM.
    5.28 +#[user@trop01]
    5.29 +
    5.30 +    createvm
    5.31 +
    5.32 +    >   INFO : Node name [Astoalith]
    5.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
    5.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
    5.35 +    >   INFO : Disc name [Astoalith.qcow]
    5.36 +    >   INFO : Disc size [16GiB]
    5.37 +
    5.38 +    vmname=Astoalith
    5.39 +
    5.40 +
    5.41 +# -----------------------------------------------------
    5.42 +# Define a hosts lookup function.
    5.43 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
    5.44 +#[user@trop01]
    5.45 +
    5.46 +    getipv4()
    5.47 +        {
    5.48 +        getent hosts "${1:?}" | cut -d ' ' -f 1
    5.49 +        }
    5.50 +
    5.51 +
    5.52 +#---------------------------------------------------------------------
    5.53 +# Update the ssh keys for the VM.
    5.54 +#[user@trop01]
    5.55 +
    5.56 +    # Remove the current key.
    5.57 +    ssh-keygen -q -R "${vmname:?}"
    5.58 +
    5.59 +    # Add the host key(s) to known_hosts
    5.60 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
    5.61 +
    5.62 +    # Add the IP address key(s) to known_hosts
    5.63 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
    5.64 +
    5.65 +
    5.66 +# -----------------------------------------------------
    5.67 +# Login as Stevedore
    5.68 +#[user@trop]
    5.69 +
    5.70 +    ssh "${vmname:?}"
    5.71 +
    5.72 +
    5.73 +# -----------------------------------------------------
    5.74 +# Install our secret function.
    5.75 +#[user@virtual]
    5.76 +
    5.77 +    mkdir "${HOME:?}/bin"
    5.78 +    cat > "${HOME:?}/bin/secret" << 'EOF'
    5.79 +#!/bin/sh
    5.80 +ssh -n \
    5.81 +    'dave@shepseskaf.roe.ac.uk' \
    5.82 +    "bin/secret '${1}'"
    5.83 +EOF
    5.84 +
    5.85 +    chmod u+x "${HOME:?}/bin/secret"
    5.86 +    secret 'frog'
    5.87 +
    5.88 +
    5.89 +# -----------------------------------------------------
    5.90 +# Download our builder compose file
    5.91 +#[user@virtual]
    5.92 +
    5.93 +    wget -O builder.yml \
    5.94 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
    5.95 +
    5.96 +
    5.97 +# -----------------------------------------------------
    5.98 +# Set the target branch
    5.99 +#[user@virtual]
   5.100 +
   5.101 +    branch=2.1.23-zrq-update-depends
   5.102 +    branch=2.1.24-zrq-userdata-drop
   5.103 +    branch=2.1.25-zrq-tap-controller
   5.104 +    branch=2.1.31-stv-cleanup
   5.105 +    branch=2.1.32-zrq-thread-pools
   5.106 +
   5.107 +
   5.108 +# -----------------------------------------------------
   5.109 +# Run our builder.
   5.110 +#[user@virtual]
   5.111 +
   5.112 +    export branch
   5.113 +    export secretsh=$(which secret)
   5.114 +
   5.115 +    docker-compose \
   5.116 +        --file "builder.yml" \
   5.117 +        run \
   5.118 +            builder
   5.119 +
   5.120 +    # -----------------------------------------------------
   5.121 +    # Test our secret function.
   5.122 +    #[root@builder]
   5.123 +
   5.124 +        secret 'frog'
   5.125 +
   5.126 +    # -----------------------------------------------------
   5.127 +    # Initialise our paths.
   5.128 +    #[root@builder]
   5.129 +
   5.130 +        PATH=${PATH}:/builder/bin
   5.131 +
   5.132 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   5.133 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   5.134 +
   5.135 +        export FIRETHORN_HOME
   5.136 +        export FIRETHORN_CODE
   5.137 +
   5.138 +    # -----------------------------------------------------
   5.139 +    # Checkout a copy of our source code.
   5.140 +    #[root@builder]
   5.141 +
   5.142 +        02.01-checkout.sh
   5.143 +
   5.144 +    # -----------------------------------------------------
   5.145 +    # Get the buildtag from our branch.
   5.146 +    #[root@builder]
   5.147 +
   5.148 +        pushd "${FIRETHORN_CODE:?}"
   5.149 +
   5.150 +            source 'bin/util.sh'
   5.151 +            export buildtag=$(getbuildtag)
   5.152 +
   5.153 +        popd
   5.154 +
   5.155 +
   5.156 +
   5.157 +#
   5.158 +# OGSA-DAI webapp fails to initialize.
   5.159 +# Lots of ArrayIndexOutOfBoundsException ..
   5.160 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
   5.161 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   5.162 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
   5.163 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   5.164 +
   5.165 +    # -----------------------------------------------------
   5.166 +    # Revert the base image ?
   5.167 +    #[root@builder]
   5.168 +
   5.169 +        vi /var/local/build/firethorn/docker/fedora/Dockerfile
   5.170 +
   5.171 +        -   # FROM latest (fedora:29)
   5.172 +        -   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   5.173 +        -   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   5.174 +
   5.175 +        +   # FROM fedora:28
   5.176 +        +   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
   5.177 +        +   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   5.178 +
   5.179 +
   5.180 +    # -----------------------------------------------------
   5.181 +    # Revert the Tomcat version ?
   5.182 +    #[root@builder]
   5.183 +
   5.184 +        vi /var/local/build/firethorn/docker/tomcat/Dockerfile
   5.185 +
   5.186 +        -   ENV TOMCAT_VERSION 9.0.20
   5.187 +        +   ENV TOMCAT_VERSION 9.0.10
   5.188 +
   5.189 +
   5.190 +        vi /var/local/build/firethorn/pom.xml
   5.191 +
   5.192 +        -   <tomcat.version>9.0.20</tomcat.version>
   5.193 +        +   <tomcat.version>9.0.20</tomcat.version>
   5.194 +
   5.195 +
   5.196 +    # -----------------------------------------------------
   5.197 +    # Revert the Java dependencies ?
   5.198 +    #[root@builder]
   5.199 +
   5.200 +        vi /var/local/build/firethorn/pom.xml
   5.201 +
   5.202 +            <dependency>
   5.203 +                <groupId>commons-codec</groupId>
   5.204 +                <artifactId>commons-codec</artifactId>
   5.205 +        -       <version>1.12</version>
   5.206 +        +       <version>1.11</version>
   5.207 +            </dependency>
   5.208 +
   5.209 +            <dependency>
   5.210 +                <groupId>org.apache.commons</groupId>
   5.211 +                <artifactId>commons-lang3</artifactId>
   5.212 +        -       <version>3.9</version>
   5.213 +        +       <version>3.7</version>
   5.214 +            </dependency>
   5.215 +
   5.216 +            <dependency>
   5.217 +                <groupId>commons-fileupload</groupId>
   5.218 +                <artifactId>commons-fileupload</artifactId>
   5.219 +        -       <version>1.4</version>
   5.220 +        +       <version>1.3.3</version>
   5.221 +            </dependency>
   5.222 +
   5.223 +            <dependency>
   5.224 +                <groupId>org.apache.httpcomponents</groupId>
   5.225 +                <artifactId>httpclient</artifactId>
   5.226 +        -       <version>4.5.8</version>
   5.227 +        +       <version>4.5.6</version>
   5.228 +            </dependency>
   5.229 +
   5.230 +            <dependency>
   5.231 +                <groupId>com.google.guava</groupId>
   5.232 +                <artifactId>guava</artifactId>
   5.233 +        -       <version>27.1-jre</version>
   5.234 +        +       <version>26.0-jre</version>
   5.235 +            </dependency>
   5.236 +
   5.237 +            <dependency>
   5.238 +                <groupId>joda-time</groupId>
   5.239 +                <artifactId>joda-time</artifactId>
   5.240 +        -       <version>2.12</version>
   5.241 +        +       <version>2.10</version>
   5.242 +            </dependency>
   5.243 +
   5.244 +            <dependency>
   5.245 +                <groupId>org.jasypt</groupId>
   5.246 +                <artifactId>jasypt</artifactId>
   5.247 +        -       <version>1.9.3</version>
   5.248 +        +       <version>1.9.2</version>
   5.249 +            </dependency>
   5.250 +
   5.251 +            <dependency>
   5.252 +                <groupId>org.projectlombok</groupId>
   5.253 +                <artifactId>lombok</artifactId>
   5.254 +        -       <version>1.18.8</version>
   5.255 +        +       <version>1.18.2</version>
   5.256 +            </dependency>
   5.257 +
   5.258 +    # -----------------------------------------------------
   5.259 +    # Revert the Spring dependencies ?
   5.260 +    #[root@builder]
   5.261 +
   5.262 +        vi /var/local/build/firethorn/firethorn-spring/pom.xml
   5.263 +
   5.264 +            <properties>
   5.265 +
   5.266 +        -       <spring.version>5.1.7.RELEASE</spring.version>
   5.267 +        +       <spring.version>5.0.8.RELEASE</spring.version>
   5.268 +
   5.269 +        -       <hibernate.version>5.4.3.Final</hibernate.version>
   5.270 +        +       <hibernate.version>5.3.1.Final</hibernate.version>
   5.271 +
   5.272 +        -       <hibernate.validator>6.0.16.Final</hibernate.validator>
   5.273 +        +       <hibernate.validator>6.0.11.Final</hibernate.validator>
   5.274 +
   5.275 +        -       <aspectj.version>1.9.4</aspectj.version>
   5.276 +        +       <aspectj.version>1.9.1</aspectj.version>
   5.277 +
   5.278 +        -       <jackson.version>2.9.9</jackson.version>
   5.279 +        +       <jackson.version>2.9.6</jackson.version>
   5.280 +
   5.281 +            </properties>
   5.282 +
   5.283 +            ....
   5.284 +            ....
   5.285 +
   5.286 +            <dependency>
   5.287 +                <groupId>cglib</groupId>
   5.288 +                <artifactId>cglib</artifactId>
   5.289 +        -       <version>3.2.12</version>
   5.290 +        +       <version>3.2.7</version>
   5.291 +            </dependency>
   5.292 +
   5.293 +
   5.294 +    # -----------------------------------------------------
   5.295 +    # Revert the logging dependencies ?
   5.296 +    #[root@builder]
   5.297 +
   5.298 +        vi /var/local/build/firethorn/firethorn-logging/pom.xml
   5.299 +
   5.300 +            <properties>
   5.301 +                ....
   5.302 +
   5.303 +        -       <org.slf4j.version>1.7.26</org.slf4j.version>
   5.304 +        +       <org.slf4j.version>1.7.25</org.slf4j.version>
   5.305 +
   5.306 +                ....
   5.307 +            </properties>
   5.308 +
   5.309 +
   5.310 +    # -----------------------------------------------------
   5.311 +    # Revert the JDBC dependencies ?
   5.312 +    #[root@builder]
   5.313 +
   5.314 +        vi /var/local/build/firethorn/firethorn-jdbc/pom.xml
   5.315 +
   5.316 +            <properties>
   5.317 +                ....
   5.318 +
   5.319 +                <dependency>
   5.320 +                    <groupId>org.hsqldb</groupId>
   5.321 +                    <artifactId>hsqldb</artifactId>
   5.322 +        -           <version>2.5.0</version>
   5.323 +        +           <version>2.4.1</version>
   5.324 +                </dependency>
   5.325 +
   5.326 +                <dependency>
   5.327 +                    <groupId>org.postgresql</groupId>
   5.328 +                    <artifactId>postgresql</artifactId>
   5.329 +        -           <version>42.2.5</version>
   5.330 +        +           <version>42.2.4</version>
   5.331 +                </dependency>
   5.332 +
   5.333 +                <dependency>
   5.334 +                    <groupId>mysql</groupId>
   5.335 +                    <artifactId>mysql-connector-java</artifactId>
   5.336 +        -           <version>8.0.16</version>
   5.337 +        +           <version>8.0.12</version>
   5.338 +                </dependency>
   5.339 +
   5.340 +                <dependency>
   5.341 +                    <groupId>org.mariadb.jdbc</groupId>
   5.342 +                    <artifactId>mariadb-java-client</artifactId>
   5.343 +        +           <version>2.4.1</version>
   5.344 +        -           <version>2.2.6</version>
   5.345 +                </dependency>
   5.346 +
   5.347 +                ....
   5.348 +            </properties>
   5.349 +
   5.350 +    # -----------------------------------------------------
   5.351 +    # Edit the webapp Thread pools.
   5.352 +    #[root@builder]
   5.353 +
   5.354 +        vi /var/local/build/firethorn/firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   5.355 +
   5.356 +            <Executor
   5.357 +                name="main-interface"
   5.358 +                namePrefix="main-interface-"
   5.359 +        -       maxThreads="20"
   5.360 +        +       maxThreads="10"
   5.361 +                minSpareThreads="5"
   5.362 +                />
   5.363 +
   5.364 +            <Executor
   5.365 +                name="callback-interface"
   5.366 +                namePrefix="callback-interface-"
   5.367 +                maxThreads="20"
   5.368 +                minSpareThreads="5"
   5.369 +                />
   5.370 +
   5.371 +
   5.372 +    # -----------------------------------------------------
   5.373 +    # Build our base images.
   5.374 +    #[root@builder]
   5.375 +
   5.376 +        04.01-buildbase.sh
   5.377 +
   5.378 +    # -----------------------------------------------------
   5.379 +    # Compile our Java code.
   5.380 +    #[root@builder]
   5.381 +
   5.382 +        05.01-javamaven.sh
   5.383 +
   5.384 +    # -----------------------------------------------------
   5.385 +    # Build our Java containers.
   5.386 +    #[root@builder]
   5.387 +
   5.388 +        05.02-javadocker.sh
   5.389 +
   5.390 +    # -----------------------------------------------------
   5.391 +    # Fetch our Python code.
   5.392 +    #[root@builder]
   5.393 +
   5.394 +        echo "Fetching client source"
   5.395 +
   5.396 +        ftpysrc='/var/local/build/client'
   5.397 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
   5.398 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
   5.399 +
   5.400 +        if [  -e "${ftpysrc:?}" ]
   5.401 +        then
   5.402 +            pushd "${ftpysrc:?}"
   5.403 +
   5.404 +                echo "Updating Python code"
   5.405 +                git pull
   5.406 +
   5.407 +            popd
   5.408 +        else
   5.409 +            pushd "$(dirname ${ftpysrc:?})"
   5.410 +
   5.411 +                echo "Cloning Python code from [${ftpygit:?}]"
   5.412 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
   5.413 +
   5.414 +            popd
   5.415 +        fi
   5.416 +
   5.417 +    # -----------------------------------------------------
   5.418 +    # Delete the embedded metadata.
   5.419 +    #[root@builder]
   5.420 +
   5.421 +        pushd "${ftpysrc:?}"
   5.422 +
   5.423 +            du -h  firethorn/meta
   5.424 +            rm -rf firethorn/meta
   5.425 +
   5.426 +        popd
   5.427 +
   5.428 +    # -----------------------------------------------------
   5.429 +    # Build our Python container.
   5.430 +    #[root@builder]
   5.431 +
   5.432 +        echo "Building client image"
   5.433 +
   5.434 +        pushd "${FIRETHORN_CODE:?}"
   5.435 +
   5.436 +            export buildsrc=$(pwd)
   5.437 +            export ftpysrc
   5.438 +
   5.439 +            docker-compose \
   5.440 +                --file docker/compose/client/firethorn-py.yml \
   5.441 +                build
   5.442 +
   5.443 +        popd
   5.444 +
   5.445 +
   5.446 +    # -----------------------------------------------------
   5.447 +    # Create our chain properties.
   5.448 +    #[user@builder]
   5.449 +
   5.450 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
   5.451 +
   5.452 +            buildtag=${buildtag:?}
   5.453 +
   5.454 +            metadata=$(pwgen 20 1)
   5.455 +            metauser=$(pwgen 20 1)
   5.456 +            metapass=$(pwgen 20 1)
   5.457 +
   5.458 +            usertype=mssql
   5.459 +            userhost=$(secret 'firethorn.user.host')
   5.460 +            userdata=$(secret 'firethorn.user.data')
   5.461 +            useruser=$(secret 'firethorn.user.user')
   5.462 +            userpass=$(secret 'firethorn.user.pass')
   5.463 +
   5.464 +            datatype=mssql
   5.465 +            datahost=$(secret 'firethorn.data.host')
   5.466 +            datadata=$(secret 'firethorn.data.data')
   5.467 +            datauser=$(secret 'firethorn.data.user')
   5.468 +            datapass=$(secret 'firethorn.data.pass')
   5.469 +
   5.470 +            tunneluser=$(secret 'ssh.tunnel.user')
   5.471 +            tunnelhost=$(secret 'ssh.tunnel.host')
   5.472 +
   5.473 +            admingroup=Hyaenidae
   5.474 +            adminuser=Aardwolf
   5.475 +            adminpass=$(pwgen 20 1)
   5.476 +
   5.477 +            guestgroup=Afrotheria
   5.478 +            guestuser=Hyrax
   5.479 +            guestpass=$(pwgen 20 1)
   5.480 +
   5.481 +            tapresource=Wilhelmina
   5.482 +            tapschemadata=data-$(pwgen 10 1)
   5.483 +            tapschemauser=user-$(pwgen 10 1)
   5.484 +            tapschemapass=pass-$(pwgen 10 1)
   5.485 +
   5.486 +EOF
   5.487 +
   5.488 +    # -----------------------------------------------------
   5.489 +    # Determine our location.
   5.490 +    #[user@builder]
   5.491 +
   5.492 +    # 'local' if VM is inside UoE
   5.493 +    # 'remote' if VM is at outside UoE
   5.494 +    # TODO Change these to 'internal' and 'external'
   5.495 +
   5.496 +        external=$(curl -4 --silent 'http://icanhazip.com/')
   5.497 +
   5.498 +        EDINBURGH='129.215.*'
   5.499 +
   5.500 +        if [[ ${external:?} == ${EDINBURGH} ]]
   5.501 +        then
   5.502 +            echo "Address [${external:?}] is in Edinburgh"
   5.503 +            #location=internal
   5.504 +            location=local
   5.505 +        else
   5.506 +            echo "Address [${external:?}] is not in Edinburgh"
   5.507 +            #location=external
   5.508 +            location=remote
   5.509 +        fi
   5.510 +
   5.511 +
   5.512 +    # -----------------------------------------------------
   5.513 +    # Link our compose config and start our test containers ...
   5.514 +    #[user@builder]
   5.515 +
   5.516 +        # Check the env file is linked !!
   5.517 +
   5.518 +        pushd "${FIRETHORN_HOME:?}"
   5.519 +
   5.520 +            ln -sf 'chain.properties' '.env'
   5.521 +
   5.522 +            docker-compose \
   5.523 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
   5.524 +                run \
   5.525 +                    angela
   5.526 +
   5.527 +
   5.528 +# -----------------------------------------------------
   5.529 +# -----------------------------------------------------
   5.530 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   5.531 +#[user@virtual]
   5.532 +
   5.533 +    sudo -s
   5.534 +
   5.535 +    container=baryptera_gillian_1
   5.536 +
   5.537 +    pushd $(
   5.538 +        docker inspect \
   5.539 +            "${container:?}" \
   5.540 +      | jq -r '
   5.541 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   5.542 +            '
   5.543 +            )
   5.544 +
   5.545 +    tail -f firethorn-debug.log
   5.546 +
   5.547 +# -----------------------------------------------------
   5.548 +# -----------------------------------------------------
   5.549 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   5.550 +#[user@virtual]
   5.551 +
   5.552 +    sudo -s
   5.553 +
   5.554 +    container=baryptera_jarmila_1
   5.555 +
   5.556 +    pushd $(
   5.557 +        docker inspect \
   5.558 +            "${container:?}" \
   5.559 +      | jq -r '
   5.560 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   5.561 +            '
   5.562 +            )
   5.563 +
   5.564 +    tail -f ogsadai.log
   5.565 +
   5.566 +# -----------------------------------------------------
   5.567 +# -----------------------------------------------------
   5.568 +# Run our Python tests ...
   5.569 +#[user@python]
   5.570 +
   5.571 +import os
   5.572 +import uuid
   5.573 +import time
   5.574 +import firethorn as ftpy
   5.575 +
   5.576 +#
   5.577 +# Create our firethorn client (using named param).
   5.578 +firethorn = ftpy.Firethorn(
   5.579 +    endpoint = os.environ.get(
   5.580 +        'endpoint'
   5.581 +        )
   5.582 +    )
   5.583 +
   5.584 +#
   5.585 +# Login as the admin account.
   5.586 +firethorn.login(
   5.587 +    os.environ.get('adminuser'),
   5.588 +    os.environ.get('adminpass'),
   5.589 +    os.environ.get('admingroup')
   5.590 +    )
   5.591 +
   5.592 +#
   5.593 +# Create a JdbcResource to connect to the ATLAS database.
   5.594 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   5.595 +    "ATLAS JDBC resource",
   5.596 +    os.environ.get('datadata'),
   5.597 +    '*',
   5.598 +    os.environ.get('datatype'),
   5.599 +    os.environ.get('datahost'),
   5.600 +    os.environ.get('datauser'),
   5.601 +    os.environ.get('datapass')
   5.602 +    )
   5.603 +print(
   5.604 +    atlas_jdbc
   5.605 +    )
   5.606 +
   5.607 +#
   5.608 +# Create an AdqlResource to represent the JdbcResource.
   5.609 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   5.610 +    "ATLAS ADQL resource"
   5.611 +    )
   5.612 +print(
   5.613 +    atlas_adql
   5.614 +    )
   5.615 +
   5.616 +#
   5.617 +# Import the target JdbcSchema into AdqlSchema.
   5.618 +schema_names = [
   5.619 +    "ATLASDR1"
   5.620 +    ]
   5.621 +
   5.622 +for schema_name in schema_names:
   5.623 +    print(schema_name)
   5.624 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   5.625 +        schema_name,
   5.626 +        "dbo"
   5.627 +        )
   5.628 +    if (None != jdbc_schema):
   5.629 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   5.630 +        adql_schema = atlas_adql.import_jdbc_schema(
   5.631 +            jdbc_schema,
   5.632 +            schema_name,
   5.633 +            metadoc=metadoc
   5.634 +            )
   5.635 +
   5.636 +#
   5.637 +# Admin user
   5.638 +# -------- -------- -------- --------
   5.639 +# Normal user
   5.640 +#
   5.641 +
   5.642 +#
   5.643 +# Login using a guest account.
   5.644 +firethorn.login(
   5.645 +    str(uuid.uuid4()),
   5.646 +    str(uuid.uuid4()),
   5.647 +    None
   5.648 +    )
   5.649 +
   5.650 +#
   5.651 +# Create a new workspace.
   5.652 +workspace = firethorn.firethorn_engine.create_adql_resource(
   5.653 +    "Query resource"
   5.654 +    )
   5.655 +
   5.656 +#
   5.657 +# Import the ATLAS schemas into our workspace
   5.658 +for schema in atlas_adql.select_schemas():
   5.659 +    workspace.import_adql_schema(
   5.660 +        schema
   5.661 +        )
   5.662 +
   5.663 +#
   5.664 +# Create and run a query.
   5.665 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   5.666 +query_obj = workspace.create_query(
   5.667 +    query_str,
   5.668 +    "COMPLETED",
   5.669 +    None,
   5.670 +    3000000
   5.671 +    )
   5.672 +print(
   5.673 +    query_obj
   5.674 +    )
   5.675 +print(
   5.676 +    query_obj.table()
   5.677 +    )
   5.678 +print(
   5.679 +    query_obj.table().count()
   5.680 +    )
   5.681 +
   5.682 +#
   5.683 +# Iterate the metadata tree
   5.684 +for schema in atlas_adql.select_schemas():
   5.685 +    for table in schema.select_tables():
   5.686 +        print(
   5.687 +            "table  [{}][{}]".format(
   5.688 +                schema.name(),
   5.689 +                table.name()
   5.690 +                )
   5.691 +            )
   5.692 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   5.693 +            schema.name(),
   5.694 +            table.name()
   5.695 +            )
   5.696 +        query_obj = workspace.create_query(
   5.697 +            query_str,
   5.698 +            "COMPLETED",
   5.699 +            None,
   5.700 +            3000000
   5.701 +            )
   5.702 +        py_table = query_obj.table().as_astropy()
   5.703 +        py_table.pprint()
   5.704 +
   5.705 +#
   5.706 +# Run some queries in parallel
   5.707 +from concurrent.futures import ThreadPoolExecutor
   5.708 +import concurrent.futures
   5.709 +from datetime import datetime
   5.710 +
   5.711 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   5.712 +
   5.713 +def do_query(workspace, query_str, limit, delay):
   5.714 +    before = datetime.now()
   5.715 +    query_obj = workspace.create_query(
   5.716 +        query_str,
   5.717 +        "COMPLETED",
   5.718 +        None,
   5.719 +        200000,
   5.720 +            {
   5.721 +            "adql.query.limit.rows"  : limit,
   5.722 +            "adql.query.delay.every" : delay
   5.723 +            }
   5.724 +        )
   5.725 +    after = datetime.now()
   5.726 +    return (
   5.727 +        (after - before),
   5.728 +        query_obj.json_object.get("results").get("count")
   5.729 +        )
   5.730 +
   5.731 +def do_queries(workspace, query_str, threads, delay):
   5.732 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   5.733 +        futures = {
   5.734 +            executor.submit(
   5.735 +                do_query,
   5.736 +                workspace,
   5.737 +                query_str,
   5.738 +                limit,
   5.739 +                delay
   5.740 +                ): limit for limit in range(threads, 0, -1)
   5.741 +            }
   5.742 +        for future in concurrent.futures.as_completed(futures):
   5.743 +            print(
   5.744 +                future.result()[0],
   5.745 +                ':',
   5.746 +                future.result()[1]
   5.747 +                )
   5.748 +
   5.749 +for loop in range(1, 4):
   5.750 +    for threads in range(1, 50):
   5.751 +        for delay in range(1000, -100, -100):
   5.752 +            print("---- ", loop, threads, delay)
   5.753 +            do_queries(
   5.754 +                workspace,
   5.755 +                query_str,
   5.756 +                threads,
   5.757 +                delay
   5.758 +                )
   5.759 +
   5.760 +
   5.761 +
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/doc/notes/zrq/20190605-01-test-build.txt	Fri Jun 07 04:40:50 2019 +0100
     6.3 @@ -0,0 +1,547 @@
     6.4 +#
     6.5 +# <meta:header>
     6.6 +#   <meta:licence>
     6.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     6.8 +#
     6.9 +#     This information is free software: you can redistribute it and/or modify
    6.10 +#     it under the terms of the GNU General Public License as published by
    6.11 +#     the Free Software Foundation, either version 3 of the License, or
    6.12 +#     (at your option) any later version.
    6.13 +#
    6.14 +#     This information is distributed in the hope that it will be useful,
    6.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    6.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    6.17 +#     GNU General Public License for more details.
    6.18 +#
    6.19 +#     You should have received a copy of the GNU General Public License
    6.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    6.21 +#   </meta:licence>
    6.22 +# </meta:header>
    6.23 +#
    6.24 +#
    6.25 +
    6.26 +# -----------------------------------------------------
    6.27 +# Login as to the VM.
    6.28 +#[user@trop]
    6.29 +
    6.30 +    ssh "${vmname:?}"
    6.31 +
    6.32 +
    6.33 +# -----------------------------------------------------
    6.34 +# Set the target branch
    6.35 +#[user@virtual]
    6.36 +
    6.37 +    branch=2.1.32-zrq-thread-pools
    6.38 +
    6.39 +
    6.40 +# -----------------------------------------------------
    6.41 +# Run our builder.
    6.42 +#[user@virtual]
    6.43 +
    6.44 +    export branch
    6.45 +    export secretsh=$(which secret)
    6.46 +
    6.47 +    docker-compose \
    6.48 +        --file "builder.yml" \
    6.49 +        run \
    6.50 +            builder
    6.51 +
    6.52 +    # -----------------------------------------------------
    6.53 +    # Test our secret function.
    6.54 +    #[root@builder]
    6.55 +
    6.56 +        secret 'frog'
    6.57 +
    6.58 +        >   Green Frog
    6.59 +
    6.60 +    # -----------------------------------------------------
    6.61 +    # Initialise our paths.
    6.62 +    #[root@builder]
    6.63 +
    6.64 +        PATH=${PATH}:/builder/bin
    6.65 +
    6.66 +        : ${FIRETHORN_HOME:=/etc/firethorn}
    6.67 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
    6.68 +
    6.69 +        export FIRETHORN_HOME
    6.70 +        export FIRETHORN_CODE
    6.71 +
    6.72 +
    6.73 +    # -----------------------------------------------------
    6.74 +    # Update our source code.
    6.75 +    #[root@builder]
    6.76 +
    6.77 +        02.01-checkout.sh
    6.78 +
    6.79 +        >   /var/local/build/firethorn /
    6.80 +        >   Updating source code from []
    6.81 +        >   pulling from http://wfau.metagrid.co.uk/code/firethorn
    6.82 +        >   searching for changes
    6.83 +        >   no changes found
    6.84 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
    6.85 +        >   /
    6.86 +        >   /var/local/build/firethorn /
    6.87 +        >   Updating branch [2.1.32-zrq-thread-pools]
    6.88 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
    6.89 +        >   /
    6.90 +
    6.91 +
    6.92 +# OGSA-DAI webapp fails to initialize.
    6.93 +# Lots of ArrayIndexOutOfBoundsException ..
    6.94 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
    6.95 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
    6.96 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
    6.97 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
    6.98 +
    6.99 +    # -----------------------------------------------------
   6.100 +    # Update the base image back to Fedora 29.
   6.101 +    # PASS that works
   6.102 +    #[root@builder]
   6.103 +
   6.104 +        pushd ${FIRETHORN_CODE:?}
   6.105 +
   6.106 +            vi docker/fedora/Dockerfile
   6.107 +
   6.108 +                # FROM latest (fedora:29)
   6.109 +                # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   6.110 +            +   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   6.111 +            -   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   6.112 +
   6.113 +
   6.114 +    # -----------------------------------------------------
   6.115 +    # Change the Tomcat version back to 9.0.20.
   6.116 +    #[root@builder]
   6.117 +
   6.118 +            vi docker/tomcat/Dockerfile
   6.119 +
   6.120 +            -   ENV TOMCAT_VERSION 9.0.10
   6.121 +            +   ENV TOMCAT_VERSION 9.0.20
   6.122 +
   6.123 +            vi pom.xml
   6.124 +
   6.125 +            -   <tomcat.version>9.0.10</tomcat.version>
   6.126 +            +   <tomcat.version>9.0.20</tomcat.version>
   6.127 +
   6.128 +#
   6.129 +# THAT causes the ArrayIndexOutOfBoundsException in OGSA-DAI.
   6.130 +2019-06-06 03:28:50,237 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
   6.131 +2019-06-06 03:28:50,572 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559791730572:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.132 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.133 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.134 +2019-06-06 03:28:50,574 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.135 +2019-06-06 03:29:12,403 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] **** inside initializeWebapp
   6.136 +2019-06-06 03:29:12,407 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@7a59a6f2: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   6.137 +
   6.138 +    # -----------------------------------------------------
   6.139 +    # Change the Tomcat version back to 9.0.10.
   6.140 +    #[root@builder]
   6.141 +
   6.142 +            vi docker/tomcat/Dockerfile
   6.143 +
   6.144 +            +   ENV TOMCAT_VERSION 9.0.10
   6.145 +            -   ENV TOMCAT_VERSION 9.0.20
   6.146 +
   6.147 +            vi pom.xml
   6.148 +
   6.149 +            +   <tomcat.version>9.0.10</tomcat.version>
   6.150 +            -   <tomcat.version>9.0.20</tomcat.version>
   6.151 +
   6.152 +    # -----------------------------------------------------
   6.153 +    # Revert the Java dependencies.
   6.154 +    #[root@builder]
   6.155 +
   6.156 +        hg status
   6.157 +
   6.158 +        >   M docker/tomcat/Dockerfile
   6.159 +        >   M pom.xml
   6.160 +        >   M firethorn-jdbc/pom.xml
   6.161 +        >   M firethorn-logging/pom.xml
   6.162 +        >   M firethorn-spring/pom.xml
   6.163 +        >   M firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   6.164 +
   6.165 +        hg revert pom.xml
   6.166 +        hg revert firethorn-jdbc/pom.xml
   6.167 +        hg revert firethorn-logging/pom.xml
   6.168 +        hg revert firethorn-spring/pom.xml
   6.169 +
   6.170 +    # -----------------------------------------------------
   6.171 +    # List the remaining changes.
   6.172 +    #[root@builder]
   6.173 +
   6.174 +        hg diff pom.xml
   6.175 +
   6.176 +            -        <tomcat.version>9.0.20</tomcat.version>
   6.177 +            +        <tomcat.version>9.0.10</tomcat.version>
   6.178 +
   6.179 +
   6.180 +        hg diff docker/tomcat/Dockerfile
   6.181 +
   6.182 +            -   ENV TOMCAT_VERSION 9.0.20
   6.183 +            +   ENV TOMCAT_VERSION 9.0.10
   6.184 +
   6.185 +
   6.186 +        hg diff firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   6.187 +
   6.188 +                 <Executor
   6.189 +                     name="main-interface"
   6.190 +                     namePrefix="main-interface-"
   6.191 +            -        maxThreads="20"
   6.192 +            +        maxThreads="10"
   6.193 +                     minSpareThreads="5"
   6.194 +                     />
   6.195 +
   6.196 +#
   6.197 +# Still get the ArrayIndexOutOfBoundsException in OGSA-DAI.
   6.198 +2019-06-06 04:09:58,633 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
   6.199 +2019-06-06 04:09:59,070 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559794199070:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.200 +2019-06-06 04:09:59,071 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.201 +2019-06-06 04:09:59,072 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.202 +2019-06-06 04:09:59,073 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   6.203 +2019-06-06 04:10:20,549 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] **** inside initializeWebapp
   6.204 +2019-06-06 04:10:20,553 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@3a7fcd66: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   6.205 +
   6.206 +
   6.207 +    # -----------------------------------------------------
   6.208 +    # Build our base images.
   6.209 +    #[root@builder]
   6.210 +
   6.211 +        04.01-buildbase.sh
   6.212 +
   6.213 +    # -----------------------------------------------------
   6.214 +    # Compile our Java code.
   6.215 +    #[root@builder]
   6.216 +
   6.217 +        05.01-javamaven.sh
   6.218 +
   6.219 +    # -----------------------------------------------------
   6.220 +    # Build our Java containers.
   6.221 +    #[root@builder]
   6.222 +
   6.223 +        05.02-javadocker.sh
   6.224 +
   6.225 +
   6.226 +    # -----------------------------------------------------
   6.227 +    # Build our Python client container.
   6.228 +    #[root@builder]
   6.229 +
   6.230 +        #
   6.231 +        # Python client container from yesterday is still in local build cache.
   6.232 +        #
   6.233 +
   6.234 +
   6.235 +    # -----------------------------------------------------
   6.236 +    # Create our chain properties.
   6.237 +    #[user@builder]
   6.238 +
   6.239 +        # Only need to do this once.
   6.240 +
   6.241 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
   6.242 +
   6.243 +            buildtag=${buildtag:?}
   6.244 +
   6.245 +            metadata=$(pwgen 20 1)
   6.246 +            metauser=$(pwgen 20 1)
   6.247 +            metapass=$(pwgen 20 1)
   6.248 +
   6.249 +            usertype=mssql
   6.250 +            userhost=$(secret 'firethorn.user.host')
   6.251 +            userdata=$(secret 'firethorn.user.data')
   6.252 +            useruser=$(secret 'firethorn.user.user')
   6.253 +            userpass=$(secret 'firethorn.user.pass')
   6.254 +
   6.255 +            datatype=mssql
   6.256 +            datahost=$(secret 'firethorn.data.host')
   6.257 +            datadata=$(secret 'firethorn.data.data')
   6.258 +            datauser=$(secret 'firethorn.data.user')
   6.259 +            datapass=$(secret 'firethorn.data.pass')
   6.260 +
   6.261 +            tunneluser=$(secret 'ssh.tunnel.user')
   6.262 +            tunnelhost=$(secret 'ssh.tunnel.host')
   6.263 +
   6.264 +            admingroup=Hyaenidae
   6.265 +            adminuser=Aardwolf
   6.266 +            adminpass=$(pwgen 20 1)
   6.267 +
   6.268 +            guestgroup=Afrotheria
   6.269 +            guestuser=Hyrax
   6.270 +            guestpass=$(pwgen 20 1)
   6.271 +
   6.272 +            tapresource=Wilhelmina
   6.273 +            tapschemadata=data-$(pwgen 10 1)
   6.274 +            tapschemauser=user-$(pwgen 10 1)
   6.275 +            tapschemapass=pass-$(pwgen 10 1)
   6.276 +
   6.277 +EOF
   6.278 +
   6.279 +    # -----------------------------------------------------
   6.280 +    # Determine our location.
   6.281 +    #[user@builder]
   6.282 +
   6.283 +        # Only need to do this once.
   6.284 +
   6.285 +        external=$(curl -4 --silent 'http://icanhazip.com/')
   6.286 +
   6.287 +        EDINBURGH='129.215.*'
   6.288 +
   6.289 +        if [[ ${external:?} == ${EDINBURGH} ]]
   6.290 +        then
   6.291 +            echo "Address [${external:?}] is in Edinburgh"
   6.292 +            #location=internal
   6.293 +            location=local
   6.294 +        else
   6.295 +            echo "Address [${external:?}] is not in Edinburgh"
   6.296 +            #location=external
   6.297 +            location=remote
   6.298 +        fi
   6.299 +
   6.300 +
   6.301 +    # -----------------------------------------------------
   6.302 +    # Link our compose config and start our test containers ...
   6.303 +    #[user@builder]
   6.304 +
   6.305 +        # Check the env file is linked !!
   6.306 +
   6.307 +        pushd "${FIRETHORN_HOME:?}"
   6.308 +
   6.309 +            ln -sf 'chain.properties' '.env'
   6.310 +
   6.311 +            docker-compose \
   6.312 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
   6.313 +                run \
   6.314 +                    angela
   6.315 +
   6.316 +
   6.317 +# -----------------------------------------------------
   6.318 +# -----------------------------------------------------
   6.319 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   6.320 +#[user@virtual]
   6.321 +
   6.322 +    sudo -s
   6.323 +
   6.324 +    container=baryptera_gillian_1
   6.325 +
   6.326 +    pushd $(
   6.327 +        docker inspect \
   6.328 +            "${container:?}" \
   6.329 +      | jq -r '
   6.330 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   6.331 +            '
   6.332 +            )
   6.333 +
   6.334 +    tail -f firethorn-debug.log
   6.335 +
   6.336 +# -----------------------------------------------------
   6.337 +# -----------------------------------------------------
   6.338 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   6.339 +#[user@virtual]
   6.340 +
   6.341 +    sudo -s
   6.342 +
   6.343 +    container=baryptera_jarmila_1
   6.344 +
   6.345 +    pushd $(
   6.346 +        docker inspect \
   6.347 +            "${container:?}" \
   6.348 +      | jq -r '
   6.349 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   6.350 +            '
   6.351 +            )
   6.352 +
   6.353 +    tail -f ogsadai.log
   6.354 +
   6.355 +# -----------------------------------------------------
   6.356 +# -----------------------------------------------------
   6.357 +# Run our Python tests ...
   6.358 +#[user@python]
   6.359 +
   6.360 +import os
   6.361 +import uuid
   6.362 +import time
   6.363 +import firethorn as ftpy
   6.364 +
   6.365 +#
   6.366 +# Create our firethorn client (using named param).
   6.367 +firethorn = ftpy.Firethorn(
   6.368 +    endpoint = os.environ.get(
   6.369 +        'endpoint'
   6.370 +        )
   6.371 +    )
   6.372 +
   6.373 +#
   6.374 +# Login as the admin account.
   6.375 +firethorn.login(
   6.376 +    os.environ.get('adminuser'),
   6.377 +    os.environ.get('adminpass'),
   6.378 +    os.environ.get('admingroup')
   6.379 +    )
   6.380 +
   6.381 +#
   6.382 +# Create a JdbcResource to connect to the ATLAS database.
   6.383 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   6.384 +    "ATLAS JDBC resource",
   6.385 +    os.environ.get('datadata'),
   6.386 +    '*',
   6.387 +    os.environ.get('datatype'),
   6.388 +    os.environ.get('datahost'),
   6.389 +    os.environ.get('datauser'),
   6.390 +    os.environ.get('datapass')
   6.391 +    )
   6.392 +print(
   6.393 +    atlas_jdbc
   6.394 +    )
   6.395 +
   6.396 +#
   6.397 +# Create an AdqlResource to represent the JdbcResource.
   6.398 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   6.399 +    "ATLAS ADQL resource"
   6.400 +    )
   6.401 +print(
   6.402 +    atlas_adql
   6.403 +    )
   6.404 +
   6.405 +#
   6.406 +# Import the target JdbcSchema into AdqlSchema.
   6.407 +schema_names = [
   6.408 +    "ATLASDR1"
   6.409 +    ]
   6.410 +
   6.411 +for schema_name in schema_names:
   6.412 +    print(schema_name)
   6.413 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   6.414 +        schema_name,
   6.415 +        "dbo"
   6.416 +        )
   6.417 +    if (None != jdbc_schema):
   6.418 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   6.419 +        adql_schema = atlas_adql.import_jdbc_schema(
   6.420 +            jdbc_schema,
   6.421 +            schema_name,
   6.422 +            metadoc=metadoc
   6.423 +            )
   6.424 +
   6.425 +#
   6.426 +# Admin user
   6.427 +# -------- -------- -------- --------
   6.428 +# Normal user
   6.429 +#
   6.430 +
   6.431 +#
   6.432 +# Login using a guest account.
   6.433 +firethorn.login(
   6.434 +    str(uuid.uuid4()),
   6.435 +    str(uuid.uuid4()),
   6.436 +    None
   6.437 +    )
   6.438 +
   6.439 +#
   6.440 +# Create a new workspace.
   6.441 +workspace = firethorn.firethorn_engine.create_adql_resource(
   6.442 +    "Query resource"
   6.443 +    )
   6.444 +
   6.445 +#
   6.446 +# Import the ATLAS schemas into our workspace
   6.447 +for schema in atlas_adql.select_schemas():
   6.448 +    workspace.import_adql_schema(
   6.449 +        schema
   6.450 +        )
   6.451 +
   6.452 +#
   6.453 +# Create and run a query.
   6.454 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   6.455 +query_obj = workspace.create_query(
   6.456 +    query_str,
   6.457 +    "COMPLETED",
   6.458 +    None,
   6.459 +    3000000
   6.460 +    )
   6.461 +print(
   6.462 +    query_obj
   6.463 +    )
   6.464 +print(
   6.465 +    query_obj.table()
   6.466 +    )
   6.467 +print(
   6.468 +    query_obj.table().count()
   6.469 +    )
   6.470 +
   6.471 +#
   6.472 +# Iterate the metadata tree
   6.473 +for schema in atlas_adql.select_schemas():
   6.474 +    for table in schema.select_tables():
   6.475 +        print(
   6.476 +            "table  [{}][{}]".format(
   6.477 +                schema.name(),
   6.478 +                table.name()
   6.479 +                )
   6.480 +            )
   6.481 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   6.482 +            schema.name(),
   6.483 +            table.name()
   6.484 +            )
   6.485 +        query_obj = workspace.create_query(
   6.486 +            query_str,
   6.487 +            "COMPLETED",
   6.488 +            None,
   6.489 +            3000000
   6.490 +            )
   6.491 +        py_table = query_obj.table().as_astropy()
   6.492 +        py_table.pprint()
   6.493 +
   6.494 +#
   6.495 +# Run some queries in parallel
   6.496 +from concurrent.futures import ThreadPoolExecutor
   6.497 +import concurrent.futures
   6.498 +from datetime import datetime
   6.499 +
   6.500 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   6.501 +
   6.502 +def do_query(workspace, query_str, limit, delay):
   6.503 +    before = datetime.now()
   6.504 +    query_obj = workspace.create_query(
   6.505 +        query_str,
   6.506 +        "COMPLETED",
   6.507 +        None,
   6.508 +        200000,
   6.509 +            {
   6.510 +            "adql.query.limit.rows"  : limit,
   6.511 +            "adql.query.delay.every" : delay
   6.512 +            }
   6.513 +        )
   6.514 +    after = datetime.now()
   6.515 +    return (
   6.516 +        (after - before),
   6.517 +        query_obj.json_object.get("results").get("count")
   6.518 +        )
   6.519 +
   6.520 +def do_queries(workspace, query_str, threads, delay):
   6.521 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   6.522 +        futures = {
   6.523 +            executor.submit(
   6.524 +                do_query,
   6.525 +                workspace,
   6.526 +                query_str,
   6.527 +                limit,
   6.528 +                delay
   6.529 +                ): limit for limit in range(threads, 0, -1)
   6.530 +            }
   6.531 +        for future in concurrent.futures.as_completed(futures):
   6.532 +            print(
   6.533 +                future.result()[0],
   6.534 +                ':',
   6.535 +                future.result()[1]
   6.536 +                )
   6.537 +
   6.538 +for loop in range(1, 10):
   6.539 +    for threads in range(1, 50):
   6.540 +        for delay in range(1000, -100, -100):
   6.541 +            print("---- ", loop, threads, delay)
   6.542 +            do_queries(
   6.543 +                workspace,
   6.544 +                query_str,
   6.545 +                threads,
   6.546 +                delay
   6.547 +                )
   6.548 +
   6.549 +
   6.550 +
     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/doc/notes/zrq/20190606-01-database-errors.txt	Fri Jun 07 04:40:50 2019 +0100
     7.3 @@ -0,0 +1,26 @@
     7.4 +#
     7.5 +# <meta:header>
     7.6 +#   <meta:licence>
     7.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     7.8 +#
     7.9 +#     This information is free software: you can redistribute it and/or modify
    7.10 +#     it under the terms of the GNU General Public License as published by
    7.11 +#     the Free Software Foundation, either version 3 of the License, or
    7.12 +#     (at your option) any later version.
    7.13 +#
    7.14 +#     This information is distributed in the hope that it will be useful,
    7.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    7.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    7.17 +#     GNU General Public License for more details.
    7.18 +#
    7.19 +#     You should have received a copy of the GNU General Public License
    7.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    7.21 +#   </meta:licence>
    7.22 +# </meta:header>
    7.23 +#
    7.24 +#
    7.25 +
    7.26 +
    7.27 +    2019-06-06 05:02:59,150 WARN  [main-interface-3] [JdbcResourceEntity] Exception while scanning catalog [53][UKIDSSDR4PLUS][Database 'UKIDSSDR4PLUS' cannot be opened due to inaccessible files or insufficient memory or disk space.  See the SQL Server errorlog for details.]
    7.28 +    2019-06-06 05:02:59,150 WARN  [main-interface-3] [SQLServerScanner] SQLException [945][S1000][Database 'UKIDSSDR4PLUS' cannot be opened due to inaccessible files or insufficient memory or disk space.  See the SQL Server errorlog for details.]
    7.29 +
     8.1 --- a/docker/apache/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
     8.2 +++ b/docker/apache/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
     8.3 @@ -17,7 +17,7 @@
     8.4  #
     8.5  # Based on example from the official httpd Docker container.
     8.6  # https://github.com/docker-library/httpd/blob/master/2.4/Dockerfile
     8.7 -FROM firethorn/fedora:2.1.32
     8.8 +FROM firethorn/fedora:2.1.33
     8.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
    8.10  
    8.11  #
     9.1 --- a/docker/builder/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
     9.2 +++ b/docker/builder/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
     9.3 @@ -15,7 +15,7 @@
     9.4  # You should have received a copy of the GNU General Public License
     9.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     9.6  
     9.7 -FROM firethorn/java:2.1.32
     9.8 +FROM firethorn/java:2.1.33
     9.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
    9.10  
    9.11  #
    10.1 --- a/docker/compose/client/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    10.2 +++ b/docker/compose/client/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    10.3 @@ -15,7 +15,7 @@
    10.4  # You should have received a copy of the GNU General Public License
    10.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    10.6  
    10.7 -FROM firethorn/tester:2.1.32
    10.8 +FROM firethorn/tester:2.1.33
    10.9  LABEL MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   10.10  
   10.11  ENV pysrc=/home/firethorn-py
    11.1 --- a/docker/compose/tests/distictella/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    11.2 +++ b/docker/compose/tests/distictella/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    11.3 @@ -15,7 +15,7 @@
    11.4  # You should have received a copy of the GNU General Public License
    11.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    11.6  
    11.7 -FROM firethorn/fedora:2.1.32
    11.8 +FROM firethorn/fedora:2.1.33
    11.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   11.10  
   11.11  RUN dnf -y install \
    12.1 --- a/docker/compose/tests/distictella/firethorn-py/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    12.2 +++ b/docker/compose/tests/distictella/firethorn-py/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    12.3 @@ -15,7 +15,7 @@
    12.4  # You should have received a copy of the GNU General Public License
    12.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    12.6  
    12.7 -FROM firethorn/fedora:2.1.32
    12.8 +FROM firethorn/fedora:2.1.33
    12.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   12.10  
   12.11  RUN dnf -y install \
    13.1 --- a/docker/fedora/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    13.2 +++ b/docker/fedora/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    13.3 @@ -15,9 +15,11 @@
    13.4  # You should have received a copy of the GNU General Public License
    13.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    13.6  
    13.7 -# FROM fedora:28
    13.8 -# https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
    13.9 +# FROM latest (fedora:29)
   13.10 +# https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   13.11 +# FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   13.12  FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   13.13 +
   13.14  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
   13.15  
   13.16  #
   13.17 @@ -35,6 +37,7 @@
   13.18  #
   13.19  # Install common admin tools.
   13.20  RUN dnf install -y \
   13.21 +    findutils  \
   13.22      gnupg \
   13.23      gzip  \
   13.24      less  \
   13.25 @@ -47,7 +50,7 @@
   13.26      zip
   13.27  
   13.28  #
   13.29 -# Add our install scripts. 
   13.30 +# Add our install scripts.
   13.31  COPY bin /usr/local/bin/
   13.32  RUN chmod a+x /usr/local/bin/*.sh
   13.33  
    14.1 --- a/docker/java/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    14.2 +++ b/docker/java/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    14.3 @@ -15,7 +15,7 @@
    14.4  # You should have received a copy of the GNU General Public License
    14.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    14.6  
    14.7 -FROM firethorn/fedora:2.1.32
    14.8 +FROM firethorn/fedora:2.1.33
    14.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   14.10  
   14.11  ENV JAVA_VERSION=12.0.0.33
    15.1 --- a/docker/postgres/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    15.2 +++ b/docker/postgres/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    15.3 @@ -15,7 +15,7 @@
    15.4  # You should have received a copy of the GNU General Public License
    15.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    15.6  
    15.7 -FROM postgres:10
    15.8 +FROM postgres:11
    15.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   15.10  
   15.11  #
    16.1 --- a/docker/python/3.4.2/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    16.2 +++ b/docker/python/3.4.2/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    16.3 @@ -1,4 +1,4 @@
    16.4 -FROM firethorn/ubuntu:2.1.32
    16.5 +FROM firethorn/ubuntu:2.1.33
    16.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    16.7  
    16.8  #
    17.1 --- a/docker/pythonlibs/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    17.2 +++ b/docker/pythonlibs/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    17.3 @@ -1,4 +1,4 @@
    17.4 -FROM firethorn/python:2.1.32
    17.5 +FROM firethorn/python:2.1.33
    17.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    17.7  
    17.8  # Install Python Libs
    18.1 --- a/docker/sql-proxy/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    18.2 +++ b/docker/sql-proxy/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    18.3 @@ -15,7 +15,7 @@
    18.4  # You should have received a copy of the GNU General Public License
    18.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    18.6  
    18.7 -FROM firethorn/fedora:2.1.32
    18.8 +FROM firethorn/fedora:2.1.33
    18.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   18.10  
   18.11  EXPOSE 1433
    19.1 --- a/docker/sql-tunnel/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    19.2 +++ b/docker/sql-tunnel/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    19.3 @@ -15,7 +15,7 @@
    19.4  # You should have received a copy of the GNU General Public License
    19.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    19.6  
    19.7 -FROM firethorn/fedora:2.1.32
    19.8 +FROM firethorn/fedora:2.1.33
    19.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   19.10  
   19.11  EXPOSE 1433
    20.1 --- a/docker/ssh-client/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    20.2 +++ b/docker/ssh-client/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    20.3 @@ -15,7 +15,7 @@
    20.4  # You should have received a copy of the GNU General Public License
    20.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    20.6  
    20.7 -FROM firethorn/fedora:2.1.32
    20.8 +FROM firethorn/fedora:2.1.33
    20.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   20.10  
   20.11  RUN dnf -y install openssh-clients
    21.1 --- a/docker/tester/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    21.2 +++ b/docker/tester/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    21.3 @@ -15,7 +15,7 @@
    21.4  # You should have received a copy of the GNU General Public License
    21.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    21.6  
    21.7 -FROM firethorn/fedora:2.1.32
    21.8 +FROM firethorn/fedora:2.1.33
    21.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   21.10  
   21.11  RUN dnf install -y jq
    22.1 --- a/docker/tomcat/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    22.2 +++ b/docker/tomcat/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    22.3 @@ -15,11 +15,10 @@
    22.4  # You should have received a copy of the GNU General Public License
    22.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    22.6  
    22.7 -FROM firethorn/java:2.1.32
    22.8 +FROM firethorn/java:2.1.33
    22.9  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
   22.10  
   22.11  # https://tomcat.apache.org/whichversion.html
   22.12 -#NV TOMCAT_VERSION 8.5.32
   22.13  ENV TOMCAT_VERSION 9.0.10
   22.14  ENV TOMCAT_MAJOR  9
   22.15  
    23.1 --- a/firethorn-admin/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    23.2 +++ b/firethorn-admin/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    23.3 @@ -32,7 +32,7 @@
    23.4      <parent>
    23.5          <groupId>uk.ac.roe.wfau</groupId>
    23.6          <artifactId>firethorn</artifactId>
    23.7 -        <version project='firethorn'>2.1.32</version>
    23.8 +        <version project='firethorn'>2.1.33</version>
    23.9      </parent>
   23.10  
   23.11      <artifactId>firethorn-admin-webapp</artifactId>
    24.1 --- a/firethorn-admin/src/main/docker/firethorn/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    24.2 +++ b/firethorn-admin/src/main/docker/firethorn/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    24.3 @@ -15,7 +15,7 @@
    24.4  # You should have received a copy of the GNU General Public License
    24.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    24.6  
    24.7 -FROM firethorn/tomcat:2.1.32
    24.8 +FROM firethorn/tomcat:2.1.33
    24.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   24.10  
   24.11  #
    25.1 --- a/firethorn-annotations/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    25.2 +++ b/firethorn-annotations/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    25.3 @@ -32,7 +32,7 @@
    25.4      <parent>
    25.5          <groupId>uk.ac.roe.wfau</groupId>
    25.6          <artifactId>firethorn</artifactId>
    25.7 -        <version project='firethorn'>2.1.32</version>
    25.8 +        <version project='firethorn'>2.1.33</version>
    25.9      </parent>
   25.10  
   25.11      <artifactId>firethorn-annotations</artifactId>
    26.1 --- a/firethorn-cds/adql/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    26.2 +++ b/firethorn-cds/adql/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    26.3 @@ -32,7 +32,7 @@
    26.4      <parent>
    26.5          <groupId>uk.ac.roe.wfau</groupId>
    26.6          <artifactId>firethorn-cds</artifactId>
    26.7 -		<version project='firethorn'>2.1.32</version>
    26.8 +		<version project='firethorn'>2.1.33</version>
    26.9  	</parent>
   26.10  
   26.11      <artifactId>firethorn-cds-adql</artifactId>
    27.1 --- a/firethorn-cds/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    27.2 +++ b/firethorn-cds/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    27.3 @@ -32,7 +32,7 @@
    27.4      <parent>
    27.5          <groupId>uk.ac.roe.wfau</groupId>
    27.6          <artifactId>firethorn</artifactId>
    27.7 -        <version project='firethorn'>2.1.32</version>
    27.8 +        <version project='firethorn'>2.1.33</version>
    27.9      </parent>
   27.10  
   27.11      <artifactId>firethorn-cds</artifactId>
    28.1 --- a/firethorn-core/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    28.2 +++ b/firethorn-core/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    28.3 @@ -32,7 +32,7 @@
    28.4      <parent>
    28.5          <groupId>uk.ac.roe.wfau</groupId>
    28.6          <artifactId>firethorn</artifactId>
    28.7 -        <version project='firethorn'>2.1.32</version>
    28.8 +        <version project='firethorn'>2.1.33</version>
    28.9  	</parent>
   28.10  
   28.11      <artifactId>firethorn-core</artifactId>
    29.1 --- a/firethorn-jdbc/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    29.2 +++ b/firethorn-jdbc/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    29.3 @@ -32,7 +32,7 @@
    29.4      <parent>
    29.5          <groupId>uk.ac.roe.wfau</groupId>
    29.6          <artifactId>firethorn</artifactId>
    29.7 -        <version project='firethorn'>2.1.32</version>
    29.8 +        <version project='firethorn'>2.1.33</version>
    29.9      </parent>
   29.10  
   29.11      <artifactId>firethorn-jdbc</artifactId>
   29.12 @@ -77,6 +77,7 @@
   29.13          <dependency>
   29.14              <groupId>org.hsqldb</groupId>
   29.15              <artifactId>hsqldb</artifactId>
   29.16 +            <!--version>2.5.0</version-->
   29.17              <version>2.4.1</version>
   29.18          </dependency>
   29.19  
   29.20 @@ -89,6 +90,7 @@
   29.21      	<dependency>
   29.22      		<groupId>org.postgresql</groupId>
   29.23      		<artifactId>postgresql</artifactId>
   29.24 +            <!--version>42.2.5</version-->
   29.25              <version>42.2.4</version>
   29.26      	</dependency>
   29.27  
   29.28 @@ -100,6 +102,7 @@
   29.29          <dependency>
   29.30              <groupId>mysql</groupId>
   29.31              <artifactId>mysql-connector-java</artifactId>
   29.32 +            <!--version>8.0.16</version-->
   29.33              <version>8.0.12</version>
   29.34          </dependency>
   29.35  
   29.36 @@ -110,6 +113,7 @@
   29.37          <dependency>
   29.38              <groupId>org.mariadb.jdbc</groupId>
   29.39              <artifactId>mariadb-java-client</artifactId>
   29.40 +            <!--version>2.4.1</version-->
   29.41              <version>2.2.6</version>
   29.42          </dependency>
   29.43  
    30.1 --- a/firethorn-logging/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    30.2 +++ b/firethorn-logging/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    30.3 @@ -32,7 +32,7 @@
    30.4      <parent>
    30.5          <groupId>uk.ac.roe.wfau</groupId>
    30.6          <artifactId>firethorn</artifactId>
    30.7 -        <version project='firethorn'>2.1.32</version>
    30.8 +        <version project='firethorn'>2.1.33</version>
    30.9      </parent>
   30.10  
   30.11      <artifactId>firethorn-logging</artifactId>
   30.12 @@ -61,7 +61,9 @@
   30.13  
   30.14          <logtype>logback</logtype>
   30.15  
   30.16 +		<!--org.slf4j.version>1.7.26</org.slf4j.version-->
   30.17  		<org.slf4j.version>1.7.25</org.slf4j.version>
   30.18 +
   30.19          <org.log4j.version>1.2.17</org.log4j.version>
   30.20          <org.logback.version>1.2.3</org.logback.version>
   30.21          <commons.logging.version>1.2</commons.logging.version>
    31.1 --- a/firethorn-ogsadai/activity/client/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    31.2 +++ b/firethorn-ogsadai/activity/client/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    31.3 @@ -32,7 +32,7 @@
    31.4      <parent>
    31.5          <groupId>uk.ac.roe.wfau</groupId>
    31.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    31.7 -        <version project='firethorn'>2.1.32</version>
    31.8 +        <version project='firethorn'>2.1.33</version>
    31.9  	</parent>
   31.10  
   31.11      <artifactId>firethorn-ogsadai-activity-client</artifactId>
    32.1 --- a/firethorn-ogsadai/activity/common/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    32.2 +++ b/firethorn-ogsadai/activity/common/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    32.3 @@ -32,7 +32,7 @@
    32.4      <parent>
    32.5          <groupId>uk.ac.roe.wfau</groupId>
    32.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    32.7 -        <version project='firethorn'>2.1.32</version>
    32.8 +        <version project='firethorn'>2.1.33</version>
    32.9      </parent>
   32.10  
   32.11      <artifactId>firethorn-ogsadai-activity-common</artifactId>
    33.1 --- a/firethorn-ogsadai/activity/context/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    33.2 +++ b/firethorn-ogsadai/activity/context/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    33.3 @@ -32,7 +32,7 @@
    33.4      <parent>
    33.5          <groupId>uk.ac.roe.wfau</groupId>
    33.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    33.7 -        <version project='firethorn'>2.1.32</version>
    33.8 +        <version project='firethorn'>2.1.33</version>
    33.9  	</parent>
   33.10  
   33.11      <artifactId>firethorn-ogsadai-activity-context</artifactId>
    34.1 --- a/firethorn-ogsadai/activity/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    34.2 +++ b/firethorn-ogsadai/activity/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    34.3 @@ -37,7 +37,7 @@
    34.4      <parent>
    34.5          <groupId>uk.ac.roe.wfau</groupId>
    34.6          <artifactId>firethorn-ogsadai</artifactId>
    34.7 -        <version project='firethorn'>2.1.32</version>
    34.8 +        <version project='firethorn'>2.1.33</version>
    34.9      </parent>
   34.10  
   34.11      <artifactId>firethorn-ogsadai-activity</artifactId>
    35.1 --- a/firethorn-ogsadai/activity/server/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    35.2 +++ b/firethorn-ogsadai/activity/server/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    35.3 @@ -32,7 +32,7 @@
    35.4      <parent>
    35.5          <groupId>uk.ac.roe.wfau</groupId>
    35.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    35.7 -        <version project='firethorn'>2.1.32</version>
    35.8 +        <version project='firethorn'>2.1.33</version>
    35.9      </parent>
   35.10  
   35.11      <artifactId>firethorn-ogsadai-activity-server</artifactId>
    36.1 --- a/firethorn-ogsadai/astro/client/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    36.2 +++ b/firethorn-ogsadai/astro/client/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    36.3 @@ -32,7 +32,7 @@
    36.4      <parent>
    36.5          <groupId>uk.ac.roe.wfau</groupId>
    36.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    36.7 -        <version project='firethorn'>2.1.32</version>
    36.8 +        <version project='firethorn'>2.1.33</version>
    36.9      </parent>
   36.10  
   36.11      <artifactId>firethorn-ogsadai-astro-client</artifactId>
    37.1 --- a/firethorn-ogsadai/astro/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    37.2 +++ b/firethorn-ogsadai/astro/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    37.3 @@ -32,7 +32,7 @@
    37.4      <parent>
    37.5          <groupId>uk.ac.roe.wfau</groupId>
    37.6          <artifactId>firethorn-ogsadai</artifactId>
    37.7 -        <version project='firethorn'>2.1.32</version>
    37.8 +        <version project='firethorn'>2.1.33</version>
    37.9      </parent>
   37.10  
   37.11      <artifactId>firethorn-ogsadai-astro</artifactId>
    38.1 --- a/firethorn-ogsadai/astro/server/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    38.2 +++ b/firethorn-ogsadai/astro/server/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    38.3 @@ -32,7 +32,7 @@
    38.4      <parent>
    38.5          <groupId>uk.ac.roe.wfau</groupId>
    38.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    38.7 -        <version project='firethorn'>2.1.32</version>
    38.8 +        <version project='firethorn'>2.1.33</version>
    38.9      </parent>
   38.10  
   38.11      <artifactId>firethorn-ogsadai-astro-server</artifactId>
    39.1 --- a/firethorn-ogsadai/depends/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    39.2 +++ b/firethorn-ogsadai/depends/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    39.3 @@ -32,7 +32,7 @@
    39.4      <parent>
    39.5          <groupId>uk.ac.roe.wfau</groupId>
    39.6          <artifactId>firethorn-ogsadai</artifactId>
    39.7 -        <version project='firethorn'>2.1.32</version>
    39.8 +        <version project='firethorn'>2.1.33</version>
    39.9      </parent>
   39.10  
   39.11      <artifactId>firethorn-ogsadai-depends</artifactId>
    40.1 --- a/firethorn-ogsadai/distributed/client/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    40.2 +++ b/firethorn-ogsadai/distributed/client/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    40.3 @@ -32,7 +32,7 @@
    40.4      <parent>
    40.5          <groupId>uk.ac.roe.wfau</groupId>
    40.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    40.7 -        <version project='firethorn'>2.1.32</version>
    40.8 +        <version project='firethorn'>2.1.33</version>
    40.9      </parent>
   40.10  
   40.11      <artifactId>firethorn-ogsadai-dqp-client</artifactId>
    41.1 --- a/firethorn-ogsadai/distributed/common/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    41.2 +++ b/firethorn-ogsadai/distributed/common/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    41.3 @@ -32,7 +32,7 @@
    41.4      <parent>
    41.5          <groupId>uk.ac.roe.wfau</groupId>
    41.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    41.7 -        <version project='firethorn'>2.1.32</version>
    41.8 +        <version project='firethorn'>2.1.33</version>
    41.9      </parent>
   41.10  
   41.11      <artifactId>firethorn-ogsadai-dqp-common</artifactId>
    42.1 --- a/firethorn-ogsadai/distributed/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    42.2 +++ b/firethorn-ogsadai/distributed/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    42.3 @@ -32,7 +32,7 @@
    42.4      <parent>
    42.5          <groupId>uk.ac.roe.wfau</groupId>
    42.6          <artifactId>firethorn-ogsadai</artifactId>
    42.7 -        <version project='firethorn'>2.1.32</version>
    42.8 +        <version project='firethorn'>2.1.33</version>
    42.9      </parent>
   42.10  
   42.11      <artifactId>firethorn-ogsadai-dqp</artifactId>
    43.1 --- a/firethorn-ogsadai/distributed/server/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    43.2 +++ b/firethorn-ogsadai/distributed/server/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    43.3 @@ -32,7 +32,7 @@
    43.4      <parent>
    43.5          <groupId>uk.ac.roe.wfau</groupId>
    43.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    43.7 -        <version project='firethorn'>2.1.32</version>
    43.8 +        <version project='firethorn'>2.1.33</version>
    43.9      </parent>
   43.10  
   43.11      <artifactId>firethorn-ogsadai-dqp-server</artifactId>
    44.1 --- a/firethorn-ogsadai/firethorn/client/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    44.2 +++ b/firethorn-ogsadai/firethorn/client/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    44.3 @@ -32,7 +32,7 @@
    44.4      <parent>
    44.5          <groupId>uk.ac.roe.wfau</groupId>
    44.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    44.7 -        <version project='firethorn'>2.1.32</version>
    44.8 +        <version project='firethorn'>2.1.33</version>
    44.9      </parent>
   44.10  
   44.11      <artifactId>firethorn-ogsadai-firethorn-client</artifactId>
    45.1 --- a/firethorn-ogsadai/firethorn/common/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    45.2 +++ b/firethorn-ogsadai/firethorn/common/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    45.3 @@ -32,7 +32,7 @@
    45.4      <parent>
    45.5          <groupId>uk.ac.roe.wfau</groupId>
    45.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    45.7 -        <version project='firethorn'>2.1.32</version>
    45.8 +        <version project='firethorn'>2.1.33</version>
    45.9      </parent>
   45.10  
   45.11      <artifactId>firethorn-ogsadai-firethorn-common</artifactId>
    46.1 --- a/firethorn-ogsadai/firethorn/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    46.2 +++ b/firethorn-ogsadai/firethorn/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    46.3 @@ -32,7 +32,7 @@
    46.4      <parent>
    46.5          <groupId>uk.ac.roe.wfau</groupId>
    46.6          <artifactId>firethorn-ogsadai</artifactId>
    46.7 -        <version project='firethorn'>2.1.32</version>
    46.8 +        <version project='firethorn'>2.1.33</version>
    46.9      </parent>
   46.10  
   46.11      <artifactId>firethorn-ogsadai-firethorn</artifactId>
    47.1 --- a/firethorn-ogsadai/firethorn/server/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    47.2 +++ b/firethorn-ogsadai/firethorn/server/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    47.3 @@ -32,7 +32,7 @@
    47.4      <parent>
    47.5          <groupId>uk.ac.roe.wfau</groupId>
    47.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    47.7 -        <version project='firethorn'>2.1.32</version>
    47.8 +        <version project='firethorn'>2.1.33</version>
    47.9      </parent>
   47.10  
   47.11      <artifactId>firethorn-ogsadai-firethorn-server</artifactId>
    48.1 --- a/firethorn-ogsadai/metadata/client/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    48.2 +++ b/firethorn-ogsadai/metadata/client/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    48.3 @@ -32,7 +32,7 @@
    48.4      <parent>
    48.5          <groupId>uk.ac.roe.wfau</groupId>
    48.6          <artifactId>firethorn-ogsadai-metadata</artifactId>
    48.7 -        <version project='firethorn'>2.1.32</version>
    48.8 +        <version project='firethorn'>2.1.33</version>
    48.9      </parent>
   48.10  
   48.11      <artifactId>firethorn-ogsadai-metadata-client</artifactId>
    49.1 --- a/firethorn-ogsadai/metadata/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    49.2 +++ b/firethorn-ogsadai/metadata/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    49.3 @@ -32,7 +32,7 @@
    49.4      <parent>
    49.5          <groupId>uk.ac.roe.wfau</groupId>
    49.6          <artifactId>firethorn-ogsadai</artifactId>
    49.7 -        <version project='firethorn'>2.1.32</version>
    49.8 +        <version project='firethorn'>2.1.33</version>
    49.9      </parent>
   49.10  
   49.11      <artifactId>firethorn-ogsadai-metadata</artifactId>
    50.1 --- a/firethorn-ogsadai/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    50.2 +++ b/firethorn-ogsadai/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    50.3 @@ -37,7 +37,7 @@
    50.4      <parent>
    50.5          <groupId>uk.ac.roe.wfau</groupId>
    50.6          <artifactId>firethorn</artifactId>
    50.7 -        <version project='firethorn'>2.1.32</version>
    50.8 +        <version project='firethorn'>2.1.33</version>
    50.9      </parent>
   50.10  
   50.11      <artifactId>firethorn-ogsadai</artifactId>
    51.1 --- a/firethorn-ogsadai/webapp/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    51.2 +++ b/firethorn-ogsadai/webapp/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    51.3 @@ -32,7 +32,7 @@
    51.4      <parent>
    51.5          <groupId>uk.ac.roe.wfau</groupId>
    51.6          <artifactId>firethorn-ogsadai</artifactId>
    51.7 -        <version project='firethorn'>2.1.32</version>
    51.8 +        <version project='firethorn'>2.1.33</version>
    51.9      </parent>
   51.10  
   51.11      <artifactId>firethorn-ogsadai-webapp</artifactId>
    52.1 --- a/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    52.2 +++ b/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    52.3 @@ -15,7 +15,7 @@
    52.4  # You should have received a copy of the GNU General Public License
    52.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    52.6  
    52.7 -FROM firethorn/tomcat:2.1.32
    52.8 +FROM firethorn/tomcat:2.1.33
    52.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   52.10  
   52.11  #
    53.1 --- a/firethorn-spring/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    53.2 +++ b/firethorn-spring/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    53.3 @@ -32,7 +32,7 @@
    53.4      <parent>
    53.5          <groupId>uk.ac.roe.wfau</groupId>
    53.6          <artifactId>firethorn</artifactId>
    53.7 -        <version project='firethorn'>2.1.32</version>
    53.8 +        <version project='firethorn'>2.1.33</version>
    53.9      </parent>
   53.10  
   53.11      <artifactId>firethorn-spring</artifactId>
   53.12 @@ -49,11 +49,19 @@
   53.13  
   53.14      <properties>
   53.15  
   53.16 +		<!--spring.version>5.1.7.RELEASE</spring.version-->
   53.17  		<spring.version>5.0.8.RELEASE</spring.version>
   53.18 +
   53.19 +        <!--hibernate.version>5.4.3.Final</hibernate.version-->
   53.20          <hibernate.version>5.3.1.Final</hibernate.version>
   53.21 +
   53.22 +	    <!--hibernate.validator>6.0.16.Final</hibernate.validator-->
   53.23  	    <hibernate.validator>6.0.11.Final</hibernate.validator>
   53.24 +
   53.25 +		<!--aspectj.version>1.9.4</aspectj.version-->
   53.26  		<aspectj.version>1.9.1</aspectj.version>
   53.27  
   53.28 +        <!--jackson.version>2.9.9</jackson.version-->
   53.29          <jackson.version>2.9.6</jackson.version>
   53.30  
   53.31      </properties>
   53.32 @@ -147,6 +155,7 @@
   53.33          <dependency>
   53.34              <groupId>cglib</groupId>
   53.35              <artifactId>cglib</artifactId>
   53.36 +            <!--version>3.2.12</version-->
   53.37              <version>3.2.7</version>
   53.38          </dependency>
   53.39  
    54.1 --- a/firethorn-stil/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    54.2 +++ b/firethorn-stil/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    54.3 @@ -32,7 +32,7 @@
    54.4      <parent>
    54.5          <groupId>uk.ac.roe.wfau</groupId>
    54.6          <artifactId>firethorn</artifactId>
    54.7 -        <version project='firethorn'>2.1.32</version>
    54.8 +        <version project='firethorn'>2.1.33</version>
    54.9      </parent>
   54.10  
   54.11      <artifactId>firethorn-stil</artifactId>
    55.1 --- a/firethorn-testing/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    55.2 +++ b/firethorn-testing/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    55.3 @@ -32,7 +32,7 @@
    55.4      <parent>
    55.5          <groupId>uk.ac.roe.wfau</groupId>
    55.6          <artifactId>firethorn</artifactId>
    55.7 -        <version project='firethorn'>2.1.32</version>
    55.8 +        <version project='firethorn'>2.1.33</version>
    55.9      </parent>
   55.10  
   55.11      <artifactId>firethorn-testing</artifactId>
    56.1 --- a/firethorn-webapp/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    56.2 +++ b/firethorn-webapp/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    56.3 @@ -32,7 +32,7 @@
    56.4      <parent>
    56.5          <groupId>uk.ac.roe.wfau</groupId>
    56.6          <artifactId>firethorn</artifactId>
    56.7 -        <version project='firethorn'>2.1.32</version>
    56.8 +        <version project='firethorn'>2.1.33</version>
    56.9      </parent>
   56.10  
   56.11      <artifactId>firethorn-webapp</artifactId>
   56.12 @@ -200,7 +200,7 @@
   56.13              <plugin>
   56.14                  <groupId>org.apache.maven.plugins</groupId>
   56.15                  <artifactId>maven-project-info-reports-plugin</artifactId>
   56.16 -                <version>2.9</version>
   56.17 +                <version>3.0.0</version>
   56.18                  <reportSets>
   56.19                      <reportSet>
   56.20                          <reports>
    57.1 --- a/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    57.2 +++ b/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    57.3 @@ -15,7 +15,7 @@
    57.4  # You should have received a copy of the GNU General Public License
    57.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    57.6  
    57.7 -FROM firethorn/tomcat:2.1.32
    57.8 +FROM firethorn/tomcat:2.1.33
    57.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   57.10  
   57.11  #
    58.1 --- a/firethorn-webapp/src/main/docker/firethorn/conf/server.xml	Mon Jun 03 04:46:00 2019 +0100
    58.2 +++ b/firethorn-webapp/src/main/docker/firethorn/conf/server.xml	Fri Jun 07 04:40:50 2019 +0100
    58.3 @@ -62,7 +62,7 @@
    58.4      <Executor
    58.5          name="main-interface"
    58.6          namePrefix="main-interface-"
    58.7 -        maxThreads="20"
    58.8 +        maxThreads="10"
    58.9          minSpareThreads="5"
   58.10          />
   58.11  
    59.1 --- a/integration/005/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    59.2 +++ b/integration/005/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    59.3 @@ -32,7 +32,7 @@
    59.4      <parent>
    59.5          <groupId>uk.ac.roe.wfau</groupId>
    59.6          <artifactId>firethorn</artifactId>
    59.7 -        <version project='firethorn'>2.1.32</version>
    59.8 +        <version project='firethorn'>2.1.33</version>
    59.9      </parent>
   59.10  
   59.11      <artifactId>firethorn-integration-tests</artifactId>
    60.1 --- a/integration/tester/Dockerfile	Mon Jun 03 04:46:00 2019 +0100
    60.2 +++ b/integration/tester/Dockerfile	Fri Jun 07 04:40:50 2019 +0100
    60.3 @@ -15,7 +15,7 @@
    60.4  # You should have received a copy of the GNU General Public License
    60.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    60.6  
    60.7 -FROM firethorn/fedora:2.1.32
    60.8 +FROM firethorn/fedora:2.1.33
    60.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   60.10  
   60.11  RUN dnf -y install perl
    61.1 --- a/pom.xml	Mon Jun 03 04:46:00 2019 +0100
    61.2 +++ b/pom.xml	Fri Jun 07 04:40:50 2019 +0100
    61.3 @@ -39,7 +39,7 @@
    61.4          | Maven project versions
    61.5          | http://www.sonatype.com/books/mvnref-book/reference/pom-relationships-sect-pom-syntax.html
    61.6          +-->
    61.7 -    <version project='firethorn'>2.1.32</version>
    61.8 +    <version project='firethorn'>2.1.33</version>
    61.9  
   61.10      <prerequisites>
   61.11          <maven>3.3.1</maven>
   61.12 @@ -295,6 +295,7 @@
   61.13              <dependency>
   61.14                  <groupId>commons-codec</groupId>
   61.15                  <artifactId>commons-codec</artifactId>
   61.16 +                <!--version>1.12</version-->
   61.17                  <version>1.11</version>
   61.18              </dependency>
   61.19              <dependency>
   61.20 @@ -315,12 +316,13 @@
   61.21              <dependency>
   61.22                  <groupId>org.apache.commons</groupId>
   61.23                  <artifactId>commons-lang3</artifactId>
   61.24 +                <!--version>3.9</version-->
   61.25                  <version>3.7</version>
   61.26              </dependency>
   61.27          	<dependency>
   61.28                  <groupId>commons-fileupload</groupId>
   61.29                  <artifactId>commons-fileupload</artifactId>
   61.30 -                <version>1.3.3</version>
   61.31 +                <version>1.4</version>
   61.32          	</dependency>
   61.33              <dependency>
   61.34                  <groupId>commons-io</groupId>
   61.35 @@ -334,6 +336,7 @@
   61.36              <dependency>
   61.37                  <groupId>org.apache.httpcomponents</groupId>
   61.38                  <artifactId>httpclient</artifactId>
   61.39 +                <!--version>4.5.8</version-->
   61.40                  <version>4.5.6</version>
   61.41              </dependency>
   61.42  
   61.43 @@ -344,6 +347,7 @@
   61.44              <dependency>
   61.45                  <groupId>com.google.guava</groupId>
   61.46                  <artifactId>guava</artifactId>
   61.47 +                <!--version>27.1-jre</version-->
   61.48                  <version>26.0-jre</version>
   61.49              </dependency>
   61.50  
   61.51 @@ -393,6 +397,7 @@
   61.52              <dependency>
   61.53                  <groupId>joda-time</groupId>
   61.54                  <artifactId>joda-time</artifactId>
   61.55 +                <!--version>2.12</version-->
   61.56                  <version>2.10</version>
   61.57              </dependency>
   61.58              <dependency>
   61.59 @@ -435,6 +440,7 @@
   61.60              <dependency>
   61.61                  <groupId>org.jasypt</groupId>
   61.62                  <artifactId>jasypt</artifactId>
   61.63 +                <!--version>1.9.3</version-->
   61.64                  <version>1.9.2</version>
   61.65              </dependency>
   61.66  
   61.67 @@ -460,6 +466,7 @@
   61.68              <dependency>
   61.69                  <groupId>org.projectlombok</groupId>
   61.70                  <artifactId>lombok</artifactId>
   61.71 +                <!--version>1.18.8</version-->
   61.72                  <version>1.18.2</version>
   61.73              </dependency>
   61.74  
   61.75 @@ -524,7 +531,7 @@
   61.76  
   61.77                  <plugin>
   61.78                      <artifactId>maven-compiler-plugin</artifactId>
   61.79 -                    <version>3.7.0</version>
   61.80 +                    <version>3.8.1</version>
   61.81                      <configuration>
   61.82                          <source>${project.javaversion}</source>
   61.83                          <target>${project.javaversion}</target>
   61.84 @@ -573,17 +580,17 @@
   61.85                  <plugin>
   61.86                      <groupId>org.apache.maven.plugins</groupId>
   61.87                      <artifactId>maven-jar-plugin</artifactId>
   61.88 -                    <version>3.1.0</version>
   61.89 +                    <version>3.1.2</version>
   61.90                  </plugin>
   61.91                  <plugin>
   61.92                      <groupId>org.apache.maven.plugins</groupId>
   61.93                      <artifactId>maven-war-plugin</artifactId>
   61.94 -                    <version>3.2.2</version>
   61.95 +                    <version>3.2.3</version>
   61.96                  </plugin>
   61.97                  <plugin>
   61.98                      <groupId>org.apache.maven.plugins</groupId>
   61.99                      <artifactId>maven-source-plugin</artifactId>
  61.100 -                    <version>3.0.1</version>
  61.101 +                    <version>3.1.0</version>
  61.102                  </plugin>
  61.103                  <plugin>
  61.104                      <groupId>org.codehaus.mojo</groupId>
  61.105 @@ -599,7 +606,7 @@
  61.106                  <plugin>
  61.107                      <groupId>org.apache.maven.plugins</groupId>
  61.108                      <artifactId>maven-surefire-plugin</artifactId>
  61.109 -                    <version>2.22.0</version>
  61.110 +                    <version>2.22.2</version>
  61.111                  </plugin>
  61.112  
  61.113                  <plugin>
  61.114 @@ -902,7 +909,7 @@
  61.115              <plugin>
  61.116                  <groupId>org.apache.maven.plugins</groupId>
  61.117                  <artifactId>maven-javadoc-plugin</artifactId>
  61.118 -                <version>3.0.1</version>
  61.119 +                <version>3.1.0</version>
  61.120  
  61.121                  <configuration>
  61.122                      <detectJavaApiLink>true</detectJavaApiLink>
  61.123 @@ -962,7 +969,7 @@
  61.124              <plugin>
  61.125                  <groupId>org.apache.maven.plugins</groupId>
  61.126                  <artifactId>maven-surefire-report-plugin</artifactId>
  61.127 -                <version>2.22.0</version>
  61.128 +                <version>2.22.2</version>
  61.129              </plugin>
  61.130  
  61.131              <!--+
  61.132 @@ -989,7 +996,7 @@
  61.133              <plugin>
  61.134                  <groupId>org.apache.maven.plugins</groupId>
  61.135                  <artifactId>maven-jxr-plugin</artifactId>
  61.136 -                <version>2.5</version>
  61.137 +                <version>3.0.0</version>
  61.138                  <configuration>
  61.139                      <linkJavadoc>true</linkJavadoc>
  61.140                  </configuration>