firethorn

changeset 4249:a9db53491c82 2.1.32-zrq-thread-pools

Reverting version changes
author Dave Morris <dmr@roe.ac.uk>
date Thu Jun 06 05:31:00 2019 +0100 (2019-06-06)
parents fd4ca3093920
children 5be5fa0abfb8
files doc/notes/zrq/20190601-01-hg-merge.txt doc/notes/zrq/20190604-01-test-build.txt doc/notes/zrq/20190605-01-test-build.txt docker/tomcat/Dockerfile firethorn-jdbc/pom.xml firethorn-logging/pom.xml firethorn-spring/pom.xml pom.xml
line diff
     1.1 --- a/doc/notes/zrq/20190601-01-hg-merge.txt	Thu Jun 06 05:30:45 2019 +0100
     1.2 +++ b/doc/notes/zrq/20190601-01-hg-merge.txt	Thu Jun 06 05:31:00 2019 +0100
     1.3 @@ -99,6 +99,7 @@
     1.4      branch=2.1.24-zrq-userdata-drop
     1.5      branch=2.1.25-zrq-tap-controller
     1.6      branch=2.1.31-stv-cleanup
     1.7 +    branch=2.1.32-zrq-thread-pools
     1.8  
     1.9  
    1.10  # -----------------------------------------------------
    1.11 @@ -342,7 +343,7 @@
    1.12      >   WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
    1.13      >   Configure a credential helper to remove this warning. See
    1.14      >   https://docs.docker.com/engine/reference/commandline/login/#credentials-store
    1.15 -    >   
    1.16 +    >
    1.17      >   Login Succeeded
    1.18      >   The push refers to repository [docker.io/firethorn/fedora]
    1.19      >   8bd87b3d77b4: Pushed
    1.20 @@ -493,10 +494,10 @@
    1.21      >   File list transfer time: 0.000 seconds
    1.22      >   Total bytes sent: 130.60M
    1.23      >   Total bytes received: 5.97K
    1.24 -    >   
    1.25 +    >
    1.26      >   sent 130.60M bytes  received 5.97K bytes  13.75M bytes/sec
    1.27      >   total size is 130.54M  speedup is 1.00
    1.28 -    >   
    1.29 +    >
    1.30      >   building file list ...
    1.31      >   95 files to consider
    1.32      >   uk/org/ogsadai/ogsadai-basic-client/4.2.2198/_remote.repositories
    1.33 @@ -517,10 +518,10 @@
    1.34      >   File list transfer time: 0.000 seconds
    1.35      >   Total bytes sent: 7.33K
    1.36      >   Total bytes received: 336
    1.37 -    >   
    1.38 +    >
    1.39      >   sent 7.33K bytes  received 336 bytes  3.07K bytes/sec
    1.40      >   total size is 29.41M  speedup is 3,836.53
    1.41 -    >   
    1.42 +    >
    1.43      >   building file list ...
    1.44      >   31 files to consider
    1.45      >   uk/ac/starlink/stil-table/3.3.1/_remote.repositories
    1.46 @@ -541,10 +542,10 @@
    1.47      >   File list transfer time: 0.000 seconds
    1.48      >   Total bytes sent: 4.14K
    1.49      >   Total bytes received: 236
    1.50 -    >   
    1.51 +    >
    1.52      >   sent 4.14K bytes  received 236 bytes  8.75K bytes/sec
    1.53      >   total size is 1.23M  speedup is 281.07
    1.54 -    >   
    1.55 +    >
    1.56      >   building file list ...
    1.57      >   3868 files to consider
    1.58      >   antlr/antlr/2.7.7/_remote.repositories
    1.59 @@ -567,7 +568,7 @@
    1.60      >   File list transfer time: 0.000 seconds
    1.61      >   Total bytes sent: 401.16K
    1.62      >   Total bytes received: 22.37K
    1.63 -    >   
    1.64 +    >
    1.65      >   sent 401.16K bytes  received 22.37K bytes  19.70K bytes/sec
    1.66      >   total size is 107.40M  speedup is 253.57
    1.67  
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/zrq/20190604-01-test-build.txt	Thu Jun 06 05:31:00 2019 +0100
     2.3 @@ -0,0 +1,758 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#
    2.25 +
    2.26 +# -----------------------------------------------------
    2.27 +# Create a new VM.
    2.28 +#[user@trop01]
    2.29 +
    2.30 +    createvm
    2.31 +
    2.32 +    >   INFO : Node name [Astoalith]
    2.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
    2.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
    2.35 +    >   INFO : Disc name [Astoalith.qcow]
    2.36 +    >   INFO : Disc size [16GiB]
    2.37 +
    2.38 +    vmname=Astoalith
    2.39 +
    2.40 +
    2.41 +# -----------------------------------------------------
    2.42 +# Define a hosts lookup function.
    2.43 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
    2.44 +#[user@trop01]
    2.45 +
    2.46 +    getipv4()
    2.47 +        {
    2.48 +        getent hosts "${1:?}" | cut -d ' ' -f 1
    2.49 +        }
    2.50 +
    2.51 +
    2.52 +#---------------------------------------------------------------------
    2.53 +# Update the ssh keys for the VM.
    2.54 +#[user@trop01]
    2.55 +
    2.56 +    # Remove the current key.
    2.57 +    ssh-keygen -q -R "${vmname:?}"
    2.58 +
    2.59 +    # Add the host key(s) to known_hosts
    2.60 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
    2.61 +
    2.62 +    # Add the IP address key(s) to known_hosts
    2.63 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
    2.64 +
    2.65 +
    2.66 +# -----------------------------------------------------
    2.67 +# Login as Stevedore
    2.68 +#[user@trop]
    2.69 +
    2.70 +    ssh "${vmname:?}"
    2.71 +
    2.72 +
    2.73 +# -----------------------------------------------------
    2.74 +# Install our secret function.
    2.75 +#[user@virtual]
    2.76 +
    2.77 +    mkdir "${HOME:?}/bin"
    2.78 +    cat > "${HOME:?}/bin/secret" << 'EOF'
    2.79 +#!/bin/sh
    2.80 +ssh -n \
    2.81 +    'dave@shepseskaf.roe.ac.uk' \
    2.82 +    "bin/secret '${1}'"
    2.83 +EOF
    2.84 +
    2.85 +    chmod u+x "${HOME:?}/bin/secret"
    2.86 +    secret 'frog'
    2.87 +
    2.88 +
    2.89 +# -----------------------------------------------------
    2.90 +# Download our builder compose file
    2.91 +#[user@virtual]
    2.92 +
    2.93 +    wget -O builder.yml \
    2.94 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
    2.95 +
    2.96 +
    2.97 +# -----------------------------------------------------
    2.98 +# Set the target branch
    2.99 +#[user@virtual]
   2.100 +
   2.101 +    branch=2.1.23-zrq-update-depends
   2.102 +    branch=2.1.24-zrq-userdata-drop
   2.103 +    branch=2.1.25-zrq-tap-controller
   2.104 +    branch=2.1.31-stv-cleanup
   2.105 +    branch=2.1.32-zrq-thread-pools
   2.106 +
   2.107 +
   2.108 +# -----------------------------------------------------
   2.109 +# Run our builder.
   2.110 +#[user@virtual]
   2.111 +
   2.112 +    export branch
   2.113 +    export secretsh=$(which secret)
   2.114 +
   2.115 +    docker-compose \
   2.116 +        --file "builder.yml" \
   2.117 +        run \
   2.118 +            builder
   2.119 +
   2.120 +    # -----------------------------------------------------
   2.121 +    # Test our secret function.
   2.122 +    #[root@builder]
   2.123 +
   2.124 +        secret 'frog'
   2.125 +
   2.126 +    # -----------------------------------------------------
   2.127 +    # Initialise our paths.
   2.128 +    #[root@builder]
   2.129 +
   2.130 +        PATH=${PATH}:/builder/bin
   2.131 +
   2.132 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   2.133 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   2.134 +
   2.135 +        export FIRETHORN_HOME
   2.136 +        export FIRETHORN_CODE
   2.137 +
   2.138 +    # -----------------------------------------------------
   2.139 +    # Checkout a copy of our source code.
   2.140 +    #[root@builder]
   2.141 +
   2.142 +        02.01-checkout.sh
   2.143 +
   2.144 +    # -----------------------------------------------------
   2.145 +    # Get the buildtag from our branch.
   2.146 +    #[root@builder]
   2.147 +
   2.148 +        pushd "${FIRETHORN_CODE:?}"
   2.149 +
   2.150 +            source 'bin/util.sh'
   2.151 +            export buildtag=$(getbuildtag)
   2.152 +
   2.153 +        popd
   2.154 +
   2.155 +
   2.156 +
   2.157 +#
   2.158 +# OGSA-DAI webapp fails to initialize.
   2.159 +# Lots of ArrayIndexOutOfBoundsException ..
   2.160 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
   2.161 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   2.162 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
   2.163 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   2.164 +
   2.165 +    # -----------------------------------------------------
   2.166 +    # Revert the base image ?
   2.167 +    #[root@builder]
   2.168 +
   2.169 +        vi /var/local/build/firethorn/docker/fedora/Dockerfile
   2.170 +
   2.171 +        -   # FROM latest (fedora:29)
   2.172 +        -   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   2.173 +        -   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   2.174 +
   2.175 +        +   # FROM fedora:28
   2.176 +        +   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
   2.177 +        +   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   2.178 +
   2.179 +
   2.180 +    # -----------------------------------------------------
   2.181 +    # Revert the Tomcat version ?
   2.182 +    #[root@builder]
   2.183 +
   2.184 +        vi /var/local/build/firethorn/docker/tomcat/Dockerfile
   2.185 +
   2.186 +        -   ENV TOMCAT_VERSION 9.0.20
   2.187 +        +   ENV TOMCAT_VERSION 9.0.10
   2.188 +
   2.189 +
   2.190 +        vi /var/local/build/firethorn/pom.xml
   2.191 +
   2.192 +        -   <tomcat.version>9.0.20</tomcat.version>
   2.193 +        +   <tomcat.version>9.0.20</tomcat.version>
   2.194 +
   2.195 +
   2.196 +    # -----------------------------------------------------
   2.197 +    # Revert the Java dependencies ?
   2.198 +    #[root@builder]
   2.199 +
   2.200 +        vi /var/local/build/firethorn/pom.xml
   2.201 +
   2.202 +            <dependency>
   2.203 +                <groupId>commons-codec</groupId>
   2.204 +                <artifactId>commons-codec</artifactId>
   2.205 +        -       <version>1.12</version>
   2.206 +        +       <version>1.11</version>
   2.207 +            </dependency>
   2.208 +
   2.209 +            <dependency>
   2.210 +                <groupId>org.apache.commons</groupId>
   2.211 +                <artifactId>commons-lang3</artifactId>
   2.212 +        -       <version>3.9</version>
   2.213 +        +       <version>3.7</version>
   2.214 +            </dependency>
   2.215 +
   2.216 +            <dependency>
   2.217 +                <groupId>commons-fileupload</groupId>
   2.218 +                <artifactId>commons-fileupload</artifactId>
   2.219 +        -       <version>1.4</version>
   2.220 +        +       <version>1.3.3</version>
   2.221 +            </dependency>
   2.222 +
   2.223 +            <dependency>
   2.224 +                <groupId>org.apache.httpcomponents</groupId>
   2.225 +                <artifactId>httpclient</artifactId>
   2.226 +        -       <version>4.5.8</version>
   2.227 +        +       <version>4.5.6</version>
   2.228 +            </dependency>
   2.229 +
   2.230 +            <dependency>
   2.231 +                <groupId>com.google.guava</groupId>
   2.232 +                <artifactId>guava</artifactId>
   2.233 +        -       <version>27.1-jre</version>
   2.234 +        +       <version>26.0-jre</version>
   2.235 +            </dependency>
   2.236 +
   2.237 +            <dependency>
   2.238 +                <groupId>joda-time</groupId>
   2.239 +                <artifactId>joda-time</artifactId>
   2.240 +        -       <version>2.12</version>
   2.241 +        +       <version>2.10</version>
   2.242 +            </dependency>
   2.243 +
   2.244 +            <dependency>
   2.245 +                <groupId>org.jasypt</groupId>
   2.246 +                <artifactId>jasypt</artifactId>
   2.247 +        -       <version>1.9.3</version>
   2.248 +        +       <version>1.9.2</version>
   2.249 +            </dependency>
   2.250 +
   2.251 +            <dependency>
   2.252 +                <groupId>org.projectlombok</groupId>
   2.253 +                <artifactId>lombok</artifactId>
   2.254 +        -       <version>1.18.8</version>
   2.255 +        +       <version>1.18.2</version>
   2.256 +            </dependency>
   2.257 +
   2.258 +    # -----------------------------------------------------
   2.259 +    # Revert the Spring dependencies ?
   2.260 +    #[root@builder]
   2.261 +
   2.262 +        vi /var/local/build/firethorn/firethorn-spring/pom.xml
   2.263 +
   2.264 +            <properties>
   2.265 +
   2.266 +        -       <spring.version>5.1.7.RELEASE</spring.version>
   2.267 +        +       <spring.version>5.0.8.RELEASE</spring.version>
   2.268 +
   2.269 +        -       <hibernate.version>5.4.3.Final</hibernate.version>
   2.270 +        +       <hibernate.version>5.3.1.Final</hibernate.version>
   2.271 +
   2.272 +        -       <hibernate.validator>6.0.16.Final</hibernate.validator>
   2.273 +        +       <hibernate.validator>6.0.11.Final</hibernate.validator>
   2.274 +
   2.275 +        -       <aspectj.version>1.9.4</aspectj.version>
   2.276 +        +       <aspectj.version>1.9.1</aspectj.version>
   2.277 +
   2.278 +        -       <jackson.version>2.9.9</jackson.version>
   2.279 +        +       <jackson.version>2.9.6</jackson.version>
   2.280 +
   2.281 +            </properties>
   2.282 +
   2.283 +            ....
   2.284 +            ....
   2.285 +
   2.286 +            <dependency>
   2.287 +                <groupId>cglib</groupId>
   2.288 +                <artifactId>cglib</artifactId>
   2.289 +        -       <version>3.2.12</version>
   2.290 +        +       <version>3.2.7</version>
   2.291 +            </dependency>
   2.292 +
   2.293 +
   2.294 +    # -----------------------------------------------------
   2.295 +    # Revert the logging dependencies ?
   2.296 +    #[root@builder]
   2.297 +
   2.298 +        vi /var/local/build/firethorn/firethorn-logging/pom.xml
   2.299 +
   2.300 +            <properties>
   2.301 +                ....
   2.302 +
   2.303 +        -       <org.slf4j.version>1.7.26</org.slf4j.version>
   2.304 +        +       <org.slf4j.version>1.7.25</org.slf4j.version>
   2.305 +
   2.306 +                ....
   2.307 +            </properties>
   2.308 +
   2.309 +
   2.310 +    # -----------------------------------------------------
   2.311 +    # Revert the JDBC dependencies ?
   2.312 +    #[root@builder]
   2.313 +
   2.314 +        vi /var/local/build/firethorn/firethorn-jdbc/pom.xml
   2.315 +
   2.316 +            <properties>
   2.317 +                ....
   2.318 +
   2.319 +                <dependency>
   2.320 +                    <groupId>org.hsqldb</groupId>
   2.321 +                    <artifactId>hsqldb</artifactId>
   2.322 +        -           <version>2.5.0</version>
   2.323 +        +           <version>2.4.1</version>
   2.324 +                </dependency>
   2.325 +
   2.326 +                <dependency>
   2.327 +                    <groupId>org.postgresql</groupId>
   2.328 +                    <artifactId>postgresql</artifactId>
   2.329 +        -           <version>42.2.5</version>
   2.330 +        +           <version>42.2.4</version>
   2.331 +                </dependency>
   2.332 +
   2.333 +                <dependency>
   2.334 +                    <groupId>mysql</groupId>
   2.335 +                    <artifactId>mysql-connector-java</artifactId>
   2.336 +        -           <version>8.0.16</version>
   2.337 +        +           <version>8.0.12</version>
   2.338 +                </dependency>
   2.339 +
   2.340 +                <dependency>
   2.341 +                    <groupId>org.mariadb.jdbc</groupId>
   2.342 +                    <artifactId>mariadb-java-client</artifactId>
   2.343 +        +           <version>2.4.1</version>
   2.344 +        -           <version>2.2.6</version>
   2.345 +                </dependency>
   2.346 +
   2.347 +                ....
   2.348 +            </properties>
   2.349 +
   2.350 +    # -----------------------------------------------------
   2.351 +    # Edit the webapp Thread pools.
   2.352 +    #[root@builder]
   2.353 +
   2.354 +        vi /var/local/build/firethorn/firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   2.355 +
   2.356 +            <Executor
   2.357 +                name="main-interface"
   2.358 +                namePrefix="main-interface-"
   2.359 +        -       maxThreads="20"
   2.360 +        +       maxThreads="10"
   2.361 +                minSpareThreads="5"
   2.362 +                />
   2.363 +
   2.364 +            <Executor
   2.365 +                name="callback-interface"
   2.366 +                namePrefix="callback-interface-"
   2.367 +                maxThreads="20"
   2.368 +                minSpareThreads="5"
   2.369 +                />
   2.370 +
   2.371 +
   2.372 +    # -----------------------------------------------------
   2.373 +    # Build our base images.
   2.374 +    #[root@builder]
   2.375 +
   2.376 +        04.01-buildbase.sh
   2.377 +
   2.378 +    # -----------------------------------------------------
   2.379 +    # Compile our Java code.
   2.380 +    #[root@builder]
   2.381 +
   2.382 +        05.01-javamaven.sh
   2.383 +
   2.384 +    # -----------------------------------------------------
   2.385 +    # Build our Java containers.
   2.386 +    #[root@builder]
   2.387 +
   2.388 +        05.02-javadocker.sh
   2.389 +
   2.390 +    # -----------------------------------------------------
   2.391 +    # Fetch our Python code.
   2.392 +    #[root@builder]
   2.393 +
   2.394 +        echo "Fetching client source"
   2.395 +
   2.396 +        ftpysrc='/var/local/build/client'
   2.397 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
   2.398 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
   2.399 +
   2.400 +        if [  -e "${ftpysrc:?}" ]
   2.401 +        then
   2.402 +            pushd "${ftpysrc:?}"
   2.403 +
   2.404 +                echo "Updating Python code"
   2.405 +                git pull
   2.406 +
   2.407 +            popd
   2.408 +        else
   2.409 +            pushd "$(dirname ${ftpysrc:?})"
   2.410 +
   2.411 +                echo "Cloning Python code from [${ftpygit:?}]"
   2.412 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
   2.413 +
   2.414 +            popd
   2.415 +        fi
   2.416 +
   2.417 +    # -----------------------------------------------------
   2.418 +    # Delete the embedded metadata.
   2.419 +    #[root@builder]
   2.420 +
   2.421 +        pushd "${ftpysrc:?}"
   2.422 +
   2.423 +            du -h  firethorn/meta
   2.424 +            rm -rf firethorn/meta
   2.425 +
   2.426 +        popd
   2.427 +
   2.428 +    # -----------------------------------------------------
   2.429 +    # Build our Python container.
   2.430 +    #[root@builder]
   2.431 +
   2.432 +        echo "Building client image"
   2.433 +
   2.434 +        pushd "${FIRETHORN_CODE:?}"
   2.435 +
   2.436 +            export buildsrc=$(pwd)
   2.437 +            export ftpysrc
   2.438 +
   2.439 +            docker-compose \
   2.440 +                --file docker/compose/client/firethorn-py.yml \
   2.441 +                build
   2.442 +
   2.443 +        popd
   2.444 +
   2.445 +
   2.446 +    # -----------------------------------------------------
   2.447 +    # Create our chain properties.
   2.448 +    #[user@builder]
   2.449 +
   2.450 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
   2.451 +
   2.452 +            buildtag=${buildtag:?}
   2.453 +
   2.454 +            metadata=$(pwgen 20 1)
   2.455 +            metauser=$(pwgen 20 1)
   2.456 +            metapass=$(pwgen 20 1)
   2.457 +
   2.458 +            usertype=mssql
   2.459 +            userhost=$(secret 'firethorn.user.host')
   2.460 +            userdata=$(secret 'firethorn.user.data')
   2.461 +            useruser=$(secret 'firethorn.user.user')
   2.462 +            userpass=$(secret 'firethorn.user.pass')
   2.463 +
   2.464 +            datatype=mssql
   2.465 +            datahost=$(secret 'firethorn.data.host')
   2.466 +            datadata=$(secret 'firethorn.data.data')
   2.467 +            datauser=$(secret 'firethorn.data.user')
   2.468 +            datapass=$(secret 'firethorn.data.pass')
   2.469 +
   2.470 +            tunneluser=$(secret 'ssh.tunnel.user')
   2.471 +            tunnelhost=$(secret 'ssh.tunnel.host')
   2.472 +
   2.473 +            admingroup=Hyaenidae
   2.474 +            adminuser=Aardwolf
   2.475 +            adminpass=$(pwgen 20 1)
   2.476 +
   2.477 +            guestgroup=Afrotheria
   2.478 +            guestuser=Hyrax
   2.479 +            guestpass=$(pwgen 20 1)
   2.480 +
   2.481 +            tapresource=Wilhelmina
   2.482 +            tapschemadata=data-$(pwgen 10 1)
   2.483 +            tapschemauser=user-$(pwgen 10 1)
   2.484 +            tapschemapass=pass-$(pwgen 10 1)
   2.485 +
   2.486 +    EOF
   2.487 +
   2.488 +    # -----------------------------------------------------
   2.489 +    # Determine our location.
   2.490 +    #[user@builder]
   2.491 +
   2.492 +    # 'local' if VM is inside UoE
   2.493 +    # 'remote' if VM is at outside UoE
   2.494 +    # TODO Change these to 'internal' and 'external'
   2.495 +
   2.496 +        external=$(curl -4 --silent 'http://icanhazip.com/')
   2.497 +
   2.498 +        EDINBURGH='129.215.*'
   2.499 +
   2.500 +        if [[ ${external:?} == ${EDINBURGH} ]]
   2.501 +        then
   2.502 +            echo "Address [${external:?}] is in Edinburgh"
   2.503 +            #location=internal
   2.504 +            location=local
   2.505 +        else
   2.506 +            echo "Address [${external:?}] is not in Edinburgh"
   2.507 +            #location=external
   2.508 +            location=remote
   2.509 +        fi
   2.510 +
   2.511 +
   2.512 +    # -----------------------------------------------------
   2.513 +    # Link our compose config and start our test containers ...
   2.514 +    #[user@builder]
   2.515 +
   2.516 +        # Check the env file is linked !!
   2.517 +
   2.518 +        pushd "${FIRETHORN_HOME:?}"
   2.519 +
   2.520 +            ln -sf 'chain.properties' '.env'
   2.521 +
   2.522 +            docker-compose \
   2.523 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
   2.524 +                run \
   2.525 +                    angela
   2.526 +
   2.527 +
   2.528 +# -----------------------------------------------------
   2.529 +# -----------------------------------------------------
   2.530 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   2.531 +#[user@virtual]
   2.532 +
   2.533 +    sudo -s
   2.534 +
   2.535 +    container=baryptera_gillian_1
   2.536 +
   2.537 +    pushd $(
   2.538 +        docker inspect \
   2.539 +            "${container:?}" \
   2.540 +      | jq -r '
   2.541 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   2.542 +            '
   2.543 +            )
   2.544 +
   2.545 +    tail -f firethorn-debug.log
   2.546 +
   2.547 +# -----------------------------------------------------
   2.548 +# -----------------------------------------------------
   2.549 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   2.550 +#[user@virtual]
   2.551 +
   2.552 +    sudo -s
   2.553 +
   2.554 +    container=baryptera_jarmila_1
   2.555 +
   2.556 +    pushd $(
   2.557 +        docker inspect \
   2.558 +            "${container:?}" \
   2.559 +      | jq -r '
   2.560 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   2.561 +            '
   2.562 +            )
   2.563 +
   2.564 +    tail -f ogsadai.log
   2.565 +
   2.566 +# -----------------------------------------------------
   2.567 +# -----------------------------------------------------
   2.568 +# Run our Python tests ...
   2.569 +#[user@python]
   2.570 +
   2.571 +import os
   2.572 +import uuid
   2.573 +import time
   2.574 +import firethorn as ftpy
   2.575 +
   2.576 +#
   2.577 +# Create our firethorn client (using named param).
   2.578 +firethorn = ftpy.Firethorn(
   2.579 +    endpoint = os.environ.get(
   2.580 +        'endpoint'
   2.581 +        )
   2.582 +    )
   2.583 +
   2.584 +#
   2.585 +# Login as the admin account.
   2.586 +firethorn.login(
   2.587 +    os.environ.get('adminuser'),
   2.588 +    os.environ.get('adminpass'),
   2.589 +    os.environ.get('admingroup')
   2.590 +    )
   2.591 +
   2.592 +#
   2.593 +# Create a JdbcResource to connect to the ATLAS database.
   2.594 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   2.595 +    "ATLAS JDBC resource",
   2.596 +    os.environ.get('datadata'),
   2.597 +    '*',
   2.598 +    os.environ.get('datatype'),
   2.599 +    os.environ.get('datahost'),
   2.600 +    os.environ.get('datauser'),
   2.601 +    os.environ.get('datapass')
   2.602 +    )
   2.603 +print(
   2.604 +    atlas_jdbc
   2.605 +    )
   2.606 +
   2.607 +#
   2.608 +# Create an AdqlResource to represent the JdbcResource.
   2.609 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   2.610 +    "ATLAS ADQL resource"
   2.611 +    )
   2.612 +print(
   2.613 +    atlas_adql
   2.614 +    )
   2.615 +
   2.616 +#
   2.617 +# Import the target JdbcSchema into AdqlSchema.
   2.618 +schema_names = [
   2.619 +    "ATLASDR1"
   2.620 +    ]
   2.621 +
   2.622 +for schema_name in schema_names:
   2.623 +    print(schema_name)
   2.624 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   2.625 +        schema_name,
   2.626 +        "dbo"
   2.627 +        )
   2.628 +    if (None != jdbc_schema):
   2.629 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   2.630 +        adql_schema = atlas_adql.import_jdbc_schema(
   2.631 +            jdbc_schema,
   2.632 +            schema_name,
   2.633 +            metadoc=metadoc
   2.634 +            )
   2.635 +
   2.636 +#
   2.637 +# Admin user
   2.638 +# -------- -------- -------- --------
   2.639 +# Normal user
   2.640 +#
   2.641 +
   2.642 +#
   2.643 +# Login using a guest account.
   2.644 +firethorn.login(
   2.645 +    str(uuid.uuid4()),
   2.646 +    str(uuid.uuid4()),
   2.647 +    None
   2.648 +    )
   2.649 +
   2.650 +#
   2.651 +# Create a new workspace.
   2.652 +workspace = firethorn.firethorn_engine.create_adql_resource(
   2.653 +    "Query resource"
   2.654 +    )
   2.655 +
   2.656 +#
   2.657 +# Import the ATLAS schemas into our workspace
   2.658 +for schema in atlas_adql.select_schemas():
   2.659 +    workspace.import_adql_schema(
   2.660 +        schema
   2.661 +        )
   2.662 +
   2.663 +#
   2.664 +# Create and run a query.
   2.665 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   2.666 +query_obj = workspace.create_query(
   2.667 +    query_str,
   2.668 +    "COMPLETED",
   2.669 +    None,
   2.670 +    3000000
   2.671 +    )
   2.672 +print(
   2.673 +    query_obj
   2.674 +    )
   2.675 +print(
   2.676 +    query_obj.table()
   2.677 +    )
   2.678 +print(
   2.679 +    query_obj.table().count()
   2.680 +    )
   2.681 +
   2.682 +#
   2.683 +# Iterate the metadata tree
   2.684 +for schema in atlas_adql.select_schemas():
   2.685 +    for table in schema.select_tables():
   2.686 +        print(
   2.687 +            "table  [{}][{}]".format(
   2.688 +                schema.name(),
   2.689 +                table.name()
   2.690 +                )
   2.691 +            )
   2.692 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   2.693 +            schema.name(),
   2.694 +            table.name()
   2.695 +            )
   2.696 +        query_obj = workspace.create_query(
   2.697 +            query_str,
   2.698 +            "COMPLETED",
   2.699 +            None,
   2.700 +            3000000
   2.701 +            )
   2.702 +        py_table = query_obj.table().as_astropy()
   2.703 +        py_table.pprint()
   2.704 +
   2.705 +#
   2.706 +# Run some queries in parallel
   2.707 +from concurrent.futures import ThreadPoolExecutor
   2.708 +import concurrent.futures
   2.709 +from datetime import datetime
   2.710 +
   2.711 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   2.712 +
   2.713 +def do_query(workspace, query_str, limit, delay):
   2.714 +    before = datetime.now()
   2.715 +    query_obj = workspace.create_query(
   2.716 +        query_str,
   2.717 +        "COMPLETED",
   2.718 +        None,
   2.719 +        200000,
   2.720 +            {
   2.721 +            "adql.query.limit.rows"  : limit,
   2.722 +            "adql.query.delay.every" : delay
   2.723 +            }
   2.724 +        )
   2.725 +    after = datetime.now()
   2.726 +    return (
   2.727 +        (after - before),
   2.728 +        query_obj.json_object.get("results").get("count")
   2.729 +        )
   2.730 +
   2.731 +def do_queries(workspace, query_str, threads, delay):
   2.732 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   2.733 +        futures = {
   2.734 +            executor.submit(
   2.735 +                do_query,
   2.736 +                workspace,
   2.737 +                query_str,
   2.738 +                limit,
   2.739 +                delay
   2.740 +                ): limit for limit in range(threads, 0, -1)
   2.741 +            }
   2.742 +        for future in concurrent.futures.as_completed(futures):
   2.743 +            print(
   2.744 +                future.result()[0],
   2.745 +                ':',
   2.746 +                future.result()[1]
   2.747 +                )
   2.748 +
   2.749 +for loop in range(1, 10):
   2.750 +    for threads in range(1, 50):
   2.751 +        for delay in range(1000, -100, -100):
   2.752 +            print("---- ", loop, threads, delay)
   2.753 +            do_queries(
   2.754 +                workspace,
   2.755 +                query_str,
   2.756 +                threads,
   2.757 +                delay
   2.758 +                )
   2.759 +
   2.760 +
   2.761 +
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/doc/notes/zrq/20190605-01-test-build.txt	Thu Jun 06 05:31:00 2019 +0100
     3.3 @@ -0,0 +1,547 @@
     3.4 +#
     3.5 +# <meta:header>
     3.6 +#   <meta:licence>
     3.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     3.8 +#
     3.9 +#     This information is free software: you can redistribute it and/or modify
    3.10 +#     it under the terms of the GNU General Public License as published by
    3.11 +#     the Free Software Foundation, either version 3 of the License, or
    3.12 +#     (at your option) any later version.
    3.13 +#
    3.14 +#     This information is distributed in the hope that it will be useful,
    3.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    3.17 +#     GNU General Public License for more details.
    3.18 +#
    3.19 +#     You should have received a copy of the GNU General Public License
    3.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    3.21 +#   </meta:licence>
    3.22 +# </meta:header>
    3.23 +#
    3.24 +#
    3.25 +
    3.26 +# -----------------------------------------------------
    3.27 +# Login as to the VM.
    3.28 +#[user@trop]
    3.29 +
    3.30 +    ssh "${vmname:?}"
    3.31 +
    3.32 +
    3.33 +# -----------------------------------------------------
    3.34 +# Set the target branch
    3.35 +#[user@virtual]
    3.36 +
    3.37 +    branch=2.1.32-zrq-thread-pools
    3.38 +
    3.39 +
    3.40 +# -----------------------------------------------------
    3.41 +# Run our builder.
    3.42 +#[user@virtual]
    3.43 +
    3.44 +    export branch
    3.45 +    export secretsh=$(which secret)
    3.46 +
    3.47 +    docker-compose \
    3.48 +        --file "builder.yml" \
    3.49 +        run \
    3.50 +            builder
    3.51 +
    3.52 +    # -----------------------------------------------------
    3.53 +    # Test our secret function.
    3.54 +    #[root@builder]
    3.55 +
    3.56 +        secret 'frog'
    3.57 +
    3.58 +        >   Green Frog
    3.59 +
    3.60 +    # -----------------------------------------------------
    3.61 +    # Initialise our paths.
    3.62 +    #[root@builder]
    3.63 +
    3.64 +        PATH=${PATH}:/builder/bin
    3.65 +
    3.66 +        : ${FIRETHORN_HOME:=/etc/firethorn}
    3.67 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
    3.68 +
    3.69 +        export FIRETHORN_HOME
    3.70 +        export FIRETHORN_CODE
    3.71 +
    3.72 +
    3.73 +    # -----------------------------------------------------
    3.74 +    # Update our source code.
    3.75 +    #[root@builder]
    3.76 +
    3.77 +        02.01-checkout.sh
    3.78 +
    3.79 +        >   /var/local/build/firethorn /
    3.80 +        >   Updating source code from []
    3.81 +        >   pulling from http://wfau.metagrid.co.uk/code/firethorn
    3.82 +        >   searching for changes
    3.83 +        >   no changes found
    3.84 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
    3.85 +        >   /
    3.86 +        >   /var/local/build/firethorn /
    3.87 +        >   Updating branch [2.1.32-zrq-thread-pools]
    3.88 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
    3.89 +        >   /
    3.90 +
    3.91 +
    3.92 +# OGSA-DAI webapp fails to initialize.
    3.93 +# Lots of ArrayIndexOutOfBoundsException ..
    3.94 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
    3.95 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
    3.96 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
    3.97 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
    3.98 +
    3.99 +    # -----------------------------------------------------
   3.100 +    # Update the base image back to Fedora 29.
   3.101 +    # PASS that works
   3.102 +    #[root@builder]
   3.103 +
   3.104 +        pushd ${FIRETHORN_CODE:?}
   3.105 +
   3.106 +            vi docker/fedora/Dockerfile
   3.107 +
   3.108 +                # FROM latest (fedora:29)
   3.109 +                # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   3.110 +            +   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   3.111 +            -   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   3.112 +
   3.113 +
   3.114 +    # -----------------------------------------------------
   3.115 +    # Change the Tomcat version back to 9.0.20.
   3.116 +    #[root@builder]
   3.117 +
   3.118 +            vi docker/tomcat/Dockerfile
   3.119 +
   3.120 +            -   ENV TOMCAT_VERSION 9.0.10
   3.121 +            +   ENV TOMCAT_VERSION 9.0.20
   3.122 +
   3.123 +            vi pom.xml
   3.124 +
   3.125 +            -   <tomcat.version>9.0.10</tomcat.version>
   3.126 +            +   <tomcat.version>9.0.20</tomcat.version>
   3.127 +
   3.128 +#
   3.129 +# THAT causes the ArrayIndexOutOfBoundsException in OGSA-DAI.
   3.130 +2019-06-06 03:28:50,237 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
   3.131 +2019-06-06 03:28:50,572 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559791730572:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.132 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.133 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.134 +2019-06-06 03:28:50,574 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.135 +2019-06-06 03:29:12,403 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] **** inside initializeWebapp
   3.136 +2019-06-06 03:29:12,407 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@7a59a6f2: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   3.137 +
   3.138 +    # -----------------------------------------------------
   3.139 +    # Change the Tomcat version back to 9.0.10.
   3.140 +    #[root@builder]
   3.141 +
   3.142 +            vi docker/tomcat/Dockerfile
   3.143 +
   3.144 +            +   ENV TOMCAT_VERSION 9.0.10
   3.145 +            -   ENV TOMCAT_VERSION 9.0.20
   3.146 +
   3.147 +            vi pom.xml
   3.148 +
   3.149 +            +   <tomcat.version>9.0.10</tomcat.version>
   3.150 +            -   <tomcat.version>9.0.20</tomcat.version>
   3.151 +
   3.152 +    # -----------------------------------------------------
   3.153 +    # Revert the Java dependencies.
   3.154 +    #[root@builder]
   3.155 +
   3.156 +        hg status
   3.157 +
   3.158 +        >   M docker/tomcat/Dockerfile
   3.159 +        >   M pom.xml
   3.160 +        >   M firethorn-jdbc/pom.xml
   3.161 +        >   M firethorn-logging/pom.xml
   3.162 +        >   M firethorn-spring/pom.xml
   3.163 +        >   M firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   3.164 +
   3.165 +        hg revert pom.xml
   3.166 +        hg revert firethorn-jdbc/pom.xml
   3.167 +        hg revert firethorn-logging/pom.xml
   3.168 +        hg revert firethorn-spring/pom.xml
   3.169 +
   3.170 +    # -----------------------------------------------------
   3.171 +    # List the remaining changes.
   3.172 +    #[root@builder]
   3.173 +
   3.174 +        hg diff pom.xml
   3.175 +
   3.176 +            -        <tomcat.version>9.0.20</tomcat.version>
   3.177 +            +        <tomcat.version>9.0.10</tomcat.version>
   3.178 +
   3.179 +
   3.180 +        hg diff docker/tomcat/Dockerfile
   3.181 +
   3.182 +            -   ENV TOMCAT_VERSION 9.0.20
   3.183 +            +   ENV TOMCAT_VERSION 9.0.10
   3.184 +
   3.185 +
   3.186 +        hg diff firethorn-webapp/src/main/docker/firethorn/conf/server.xml
   3.187 +
   3.188 +                 <Executor
   3.189 +                     name="main-interface"
   3.190 +                     namePrefix="main-interface-"
   3.191 +            -        maxThreads="20"
   3.192 +            +        maxThreads="10"
   3.193 +                     minSpareThreads="5"
   3.194 +                     />
   3.195 +
   3.196 +#
   3.197 +# Still get the ArrayIndexOutOfBoundsException in OGSA-DAI.
   3.198 +2019-06-06 04:09:58,633 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
   3.199 +2019-06-06 04:09:59,070 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559794199070:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.200 +2019-06-06 04:09:59,071 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.201 +2019-06-06 04:09:59,072 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.202 +2019-06-06 04:09:59,073 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   3.203 +2019-06-06 04:10:20,549 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] **** inside initializeWebapp
   3.204 +2019-06-06 04:10:20,553 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@3a7fcd66: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   3.205 +
   3.206 +
   3.207 +    # -----------------------------------------------------
   3.208 +    # Build our base images.
   3.209 +    #[root@builder]
   3.210 +
   3.211 +        04.01-buildbase.sh
   3.212 +
   3.213 +    # -----------------------------------------------------
   3.214 +    # Compile our Java code.
   3.215 +    #[root@builder]
   3.216 +
   3.217 +        05.01-javamaven.sh
   3.218 +
   3.219 +    # -----------------------------------------------------
   3.220 +    # Build our Java containers.
   3.221 +    #[root@builder]
   3.222 +
   3.223 +        05.02-javadocker.sh
   3.224 +
   3.225 +
   3.226 +    # -----------------------------------------------------
   3.227 +    # Build our Python client container.
   3.228 +    #[root@builder]
   3.229 +
   3.230 +        #
   3.231 +        # Python client container from yesterday is still in local build cache.
   3.232 +        #
   3.233 +
   3.234 +
   3.235 +    # -----------------------------------------------------
   3.236 +    # Create our chain properties.
   3.237 +    #[user@builder]
   3.238 +
   3.239 +        # Only need to do this once.
   3.240 +
   3.241 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
   3.242 +
   3.243 +            buildtag=${buildtag:?}
   3.244 +
   3.245 +            metadata=$(pwgen 20 1)
   3.246 +            metauser=$(pwgen 20 1)
   3.247 +            metapass=$(pwgen 20 1)
   3.248 +
   3.249 +            usertype=mssql
   3.250 +            userhost=$(secret 'firethorn.user.host')
   3.251 +            userdata=$(secret 'firethorn.user.data')
   3.252 +            useruser=$(secret 'firethorn.user.user')
   3.253 +            userpass=$(secret 'firethorn.user.pass')
   3.254 +
   3.255 +            datatype=mssql
   3.256 +            datahost=$(secret 'firethorn.data.host')
   3.257 +            datadata=$(secret 'firethorn.data.data')
   3.258 +            datauser=$(secret 'firethorn.data.user')
   3.259 +            datapass=$(secret 'firethorn.data.pass')
   3.260 +
   3.261 +            tunneluser=$(secret 'ssh.tunnel.user')
   3.262 +            tunnelhost=$(secret 'ssh.tunnel.host')
   3.263 +
   3.264 +            admingroup=Hyaenidae
   3.265 +            adminuser=Aardwolf
   3.266 +            adminpass=$(pwgen 20 1)
   3.267 +
   3.268 +            guestgroup=Afrotheria
   3.269 +            guestuser=Hyrax
   3.270 +            guestpass=$(pwgen 20 1)
   3.271 +
   3.272 +            tapresource=Wilhelmina
   3.273 +            tapschemadata=data-$(pwgen 10 1)
   3.274 +            tapschemauser=user-$(pwgen 10 1)
   3.275 +            tapschemapass=pass-$(pwgen 10 1)
   3.276 +
   3.277 +EOF
   3.278 +
   3.279 +    # -----------------------------------------------------
   3.280 +    # Determine our location.
   3.281 +    #[user@builder]
   3.282 +
   3.283 +        # Only need to do this once.
   3.284 +
   3.285 +        external=$(curl -4 --silent 'http://icanhazip.com/')
   3.286 +
   3.287 +        EDINBURGH='129.215.*'
   3.288 +
   3.289 +        if [[ ${external:?} == ${EDINBURGH} ]]
   3.290 +        then
   3.291 +            echo "Address [${external:?}] is in Edinburgh"
   3.292 +            #location=internal
   3.293 +            location=local
   3.294 +        else
   3.295 +            echo "Address [${external:?}] is not in Edinburgh"
   3.296 +            #location=external
   3.297 +            location=remote
   3.298 +        fi
   3.299 +
   3.300 +
   3.301 +    # -----------------------------------------------------
   3.302 +    # Link our compose config and start our test containers ...
   3.303 +    #[user@builder]
   3.304 +
   3.305 +        # Check the env file is linked !!
   3.306 +
   3.307 +        pushd "${FIRETHORN_HOME:?}"
   3.308 +
   3.309 +            ln -sf 'chain.properties' '.env'
   3.310 +
   3.311 +            docker-compose \
   3.312 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
   3.313 +                run \
   3.314 +                    angela
   3.315 +
   3.316 +
   3.317 +# -----------------------------------------------------
   3.318 +# -----------------------------------------------------
   3.319 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   3.320 +#[user@virtual]
   3.321 +
   3.322 +    sudo -s
   3.323 +
   3.324 +    container=baryptera_gillian_1
   3.325 +
   3.326 +    pushd $(
   3.327 +        docker inspect \
   3.328 +            "${container:?}" \
   3.329 +      | jq -r '
   3.330 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   3.331 +            '
   3.332 +            )
   3.333 +
   3.334 +    tail -f firethorn-debug.log
   3.335 +
   3.336 +# -----------------------------------------------------
   3.337 +# -----------------------------------------------------
   3.338 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   3.339 +#[user@virtual]
   3.340 +
   3.341 +    sudo -s
   3.342 +
   3.343 +    container=baryptera_jarmila_1
   3.344 +
   3.345 +    pushd $(
   3.346 +        docker inspect \
   3.347 +            "${container:?}" \
   3.348 +      | jq -r '
   3.349 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   3.350 +            '
   3.351 +            )
   3.352 +
   3.353 +    tail -f ogsadai.log
   3.354 +
   3.355 +# -----------------------------------------------------
   3.356 +# -----------------------------------------------------
   3.357 +# Run our Python tests ...
   3.358 +#[user@python]
   3.359 +
   3.360 +import os
   3.361 +import uuid
   3.362 +import time
   3.363 +import firethorn as ftpy
   3.364 +
   3.365 +#
   3.366 +# Create our firethorn client (using named param).
   3.367 +firethorn = ftpy.Firethorn(
   3.368 +    endpoint = os.environ.get(
   3.369 +        'endpoint'
   3.370 +        )
   3.371 +    )
   3.372 +
   3.373 +#
   3.374 +# Login as the admin account.
   3.375 +firethorn.login(
   3.376 +    os.environ.get('adminuser'),
   3.377 +    os.environ.get('adminpass'),
   3.378 +    os.environ.get('admingroup')
   3.379 +    )
   3.380 +
   3.381 +#
   3.382 +# Create a JdbcResource to connect to the ATLAS database.
   3.383 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   3.384 +    "ATLAS JDBC resource",
   3.385 +    os.environ.get('datadata'),
   3.386 +    '*',
   3.387 +    os.environ.get('datatype'),
   3.388 +    os.environ.get('datahost'),
   3.389 +    os.environ.get('datauser'),
   3.390 +    os.environ.get('datapass')
   3.391 +    )
   3.392 +print(
   3.393 +    atlas_jdbc
   3.394 +    )
   3.395 +
   3.396 +#
   3.397 +# Create an AdqlResource to represent the JdbcResource.
   3.398 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   3.399 +    "ATLAS ADQL resource"
   3.400 +    )
   3.401 +print(
   3.402 +    atlas_adql
   3.403 +    )
   3.404 +
   3.405 +#
   3.406 +# Import the target JdbcSchema into AdqlSchema.
   3.407 +schema_names = [
   3.408 +    "ATLASDR1"
   3.409 +    ]
   3.410 +
   3.411 +for schema_name in schema_names:
   3.412 +    print(schema_name)
   3.413 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   3.414 +        schema_name,
   3.415 +        "dbo"
   3.416 +        )
   3.417 +    if (None != jdbc_schema):
   3.418 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   3.419 +        adql_schema = atlas_adql.import_jdbc_schema(
   3.420 +            jdbc_schema,
   3.421 +            schema_name,
   3.422 +            metadoc=metadoc
   3.423 +            )
   3.424 +
   3.425 +#
   3.426 +# Admin user
   3.427 +# -------- -------- -------- --------
   3.428 +# Normal user
   3.429 +#
   3.430 +
   3.431 +#
   3.432 +# Login using a guest account.
   3.433 +firethorn.login(
   3.434 +    str(uuid.uuid4()),
   3.435 +    str(uuid.uuid4()),
   3.436 +    None
   3.437 +    )
   3.438 +
   3.439 +#
   3.440 +# Create a new workspace.
   3.441 +workspace = firethorn.firethorn_engine.create_adql_resource(
   3.442 +    "Query resource"
   3.443 +    )
   3.444 +
   3.445 +#
   3.446 +# Import the ATLAS schemas into our workspace
   3.447 +for schema in atlas_adql.select_schemas():
   3.448 +    workspace.import_adql_schema(
   3.449 +        schema
   3.450 +        )
   3.451 +
   3.452 +#
   3.453 +# Create and run a query.
   3.454 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   3.455 +query_obj = workspace.create_query(
   3.456 +    query_str,
   3.457 +    "COMPLETED",
   3.458 +    None,
   3.459 +    3000000
   3.460 +    )
   3.461 +print(
   3.462 +    query_obj
   3.463 +    )
   3.464 +print(
   3.465 +    query_obj.table()
   3.466 +    )
   3.467 +print(
   3.468 +    query_obj.table().count()
   3.469 +    )
   3.470 +
   3.471 +#
   3.472 +# Iterate the metadata tree
   3.473 +for schema in atlas_adql.select_schemas():
   3.474 +    for table in schema.select_tables():
   3.475 +        print(
   3.476 +            "table  [{}][{}]".format(
   3.477 +                schema.name(),
   3.478 +                table.name()
   3.479 +                )
   3.480 +            )
   3.481 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   3.482 +            schema.name(),
   3.483 +            table.name()
   3.484 +            )
   3.485 +        query_obj = workspace.create_query(
   3.486 +            query_str,
   3.487 +            "COMPLETED",
   3.488 +            None,
   3.489 +            3000000
   3.490 +            )
   3.491 +        py_table = query_obj.table().as_astropy()
   3.492 +        py_table.pprint()
   3.493 +
   3.494 +#
   3.495 +# Run some queries in parallel
   3.496 +from concurrent.futures import ThreadPoolExecutor
   3.497 +import concurrent.futures
   3.498 +from datetime import datetime
   3.499 +
   3.500 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   3.501 +
   3.502 +def do_query(workspace, query_str, limit, delay):
   3.503 +    before = datetime.now()
   3.504 +    query_obj = workspace.create_query(
   3.505 +        query_str,
   3.506 +        "COMPLETED",
   3.507 +        None,
   3.508 +        200000,
   3.509 +            {
   3.510 +            "adql.query.limit.rows"  : limit,
   3.511 +            "adql.query.delay.every" : delay
   3.512 +            }
   3.513 +        )
   3.514 +    after = datetime.now()
   3.515 +    return (
   3.516 +        (after - before),
   3.517 +        query_obj.json_object.get("results").get("count")
   3.518 +        )
   3.519 +
   3.520 +def do_queries(workspace, query_str, threads, delay):
   3.521 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   3.522 +        futures = {
   3.523 +            executor.submit(
   3.524 +                do_query,
   3.525 +                workspace,
   3.526 +                query_str,
   3.527 +                limit,
   3.528 +                delay
   3.529 +                ): limit for limit in range(threads, 0, -1)
   3.530 +            }
   3.531 +        for future in concurrent.futures.as_completed(futures):
   3.532 +            print(
   3.533 +                future.result()[0],
   3.534 +                ':',
   3.535 +                future.result()[1]
   3.536 +                )
   3.537 +
   3.538 +for loop in range(1, 10):
   3.539 +    for threads in range(1, 50):
   3.540 +        for delay in range(1000, -100, -100):
   3.541 +            print("---- ", loop, threads, delay)
   3.542 +            do_queries(
   3.543 +                workspace,
   3.544 +                query_str,
   3.545 +                threads,
   3.546 +                delay
   3.547 +                )
   3.548 +
   3.549 +
   3.550 +
     4.1 --- a/docker/tomcat/Dockerfile	Thu Jun 06 05:30:45 2019 +0100
     4.2 +++ b/docker/tomcat/Dockerfile	Thu Jun 06 05:31:00 2019 +0100
     4.3 @@ -19,7 +19,7 @@
     4.4  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
     4.5  
     4.6  # https://tomcat.apache.org/whichversion.html
     4.7 -ENV TOMCAT_VERSION 9.0.20
     4.8 +ENV TOMCAT_VERSION 9.0.10
     4.9  ENV TOMCAT_MAJOR  9
    4.10  
    4.11  ENV TOMCAT_DIRNAME apache-tomcat-${TOMCAT_VERSION}
     5.1 --- a/firethorn-jdbc/pom.xml	Thu Jun 06 05:30:45 2019 +0100
     5.2 +++ b/firethorn-jdbc/pom.xml	Thu Jun 06 05:31:00 2019 +0100
     5.3 @@ -77,7 +77,8 @@
     5.4          <dependency>
     5.5              <groupId>org.hsqldb</groupId>
     5.6              <artifactId>hsqldb</artifactId>
     5.7 -            <version>2.5.0</version>
     5.8 +            <!--version>2.5.0</version-->
     5.9 +            <version>2.4.1</version>
    5.10          </dependency>
    5.11  
    5.12          <!--+
    5.13 @@ -89,7 +90,8 @@
    5.14      	<dependency>
    5.15      		<groupId>org.postgresql</groupId>
    5.16      		<artifactId>postgresql</artifactId>
    5.17 -            <version>42.2.5</version>
    5.18 +            <!--version>42.2.5</version-->
    5.19 +            <version>42.2.4</version>
    5.20      	</dependency>
    5.21  
    5.22          <!--+
    5.23 @@ -100,7 +102,8 @@
    5.24          <dependency>
    5.25              <groupId>mysql</groupId>
    5.26              <artifactId>mysql-connector-java</artifactId>
    5.27 -            <version>8.0.16</version>
    5.28 +            <!--version>8.0.16</version-->
    5.29 +            <version>8.0.12</version>
    5.30          </dependency>
    5.31  
    5.32          <!--+
    5.33 @@ -110,7 +113,8 @@
    5.34          <dependency>
    5.35              <groupId>org.mariadb.jdbc</groupId>
    5.36              <artifactId>mariadb-java-client</artifactId>
    5.37 -            <version>2.4.1</version>
    5.38 +            <!--version>2.4.1</version-->
    5.39 +            <version>2.2.6</version>
    5.40          </dependency>
    5.41  
    5.42          <!--+
     6.1 --- a/firethorn-logging/pom.xml	Thu Jun 06 05:30:45 2019 +0100
     6.2 +++ b/firethorn-logging/pom.xml	Thu Jun 06 05:31:00 2019 +0100
     6.3 @@ -61,7 +61,9 @@
     6.4  
     6.5          <logtype>logback</logtype>
     6.6  
     6.7 -		<org.slf4j.version>1.7.26</org.slf4j.version>
     6.8 +		<!--org.slf4j.version>1.7.26</org.slf4j.version-->
     6.9 +		<org.slf4j.version>1.7.25</org.slf4j.version>
    6.10 +
    6.11          <org.log4j.version>1.2.17</org.log4j.version>
    6.12          <org.logback.version>1.2.3</org.logback.version>
    6.13          <commons.logging.version>1.2</commons.logging.version>
     7.1 --- a/firethorn-spring/pom.xml	Thu Jun 06 05:30:45 2019 +0100
     7.2 +++ b/firethorn-spring/pom.xml	Thu Jun 06 05:31:00 2019 +0100
     7.3 @@ -49,12 +49,20 @@
     7.4  
     7.5      <properties>
     7.6  
     7.7 -		<spring.version>5.1.7.RELEASE</spring.version>
     7.8 -        <hibernate.version>5.4.3.Final</hibernate.version>
     7.9 -	    <hibernate.validator>6.0.16.Final</hibernate.validator>
    7.10 -		<aspectj.version>1.9.4</aspectj.version>
    7.11 +		<!--spring.version>5.1.7.RELEASE</spring.version-->
    7.12 +		<spring.version>5.0.8.RELEASE</spring.version>
    7.13  
    7.14 -        <jackson.version>2.9.9</jackson.version>
    7.15 +        <!--hibernate.version>5.4.3.Final</hibernate.version-->
    7.16 +        <hibernate.version>5.3.1.Final</hibernate.version>
    7.17 +
    7.18 +	    <!--hibernate.validator>6.0.16.Final</hibernate.validator-->
    7.19 +	    <hibernate.validator>6.0.11.Final</hibernate.validator>
    7.20 +
    7.21 +		<!--aspectj.version>1.9.4</aspectj.version-->
    7.22 +		<aspectj.version>1.9.1</aspectj.version>
    7.23 +
    7.24 +        <!--jackson.version>2.9.9</jackson.version-->
    7.25 +        <jackson.version>2.9.6</jackson.version>
    7.26  
    7.27      </properties>
    7.28  
    7.29 @@ -147,7 +155,8 @@
    7.30          <dependency>
    7.31              <groupId>cglib</groupId>
    7.32              <artifactId>cglib</artifactId>
    7.33 -            <version>3.2.12</version>
    7.34 +            <!--version>3.2.12</version-->
    7.35 +            <version>3.2.7</version>
    7.36          </dependency>
    7.37  
    7.38          <!--+
     8.1 --- a/pom.xml	Thu Jun 06 05:30:45 2019 +0100
     8.2 +++ b/pom.xml	Thu Jun 06 05:31:00 2019 +0100
     8.3 @@ -207,7 +207,7 @@
     8.4              | Tomcat version.
     8.5              | https://tomcat.apache.org/whichversion.html
     8.6              +-->
     8.7 -        <tomcat.version>9.0.20</tomcat.version>
     8.8 +        <tomcat.version>9.0.10</tomcat.version>
     8.9  
    8.10      </properties>
    8.11  
    8.12 @@ -295,7 +295,8 @@
    8.13              <dependency>
    8.14                  <groupId>commons-codec</groupId>
    8.15                  <artifactId>commons-codec</artifactId>
    8.16 -                <version>1.12</version>
    8.17 +                <!--version>1.12</version-->
    8.18 +                <version>1.11</version>
    8.19              </dependency>
    8.20              <dependency>
    8.21                  <groupId>commons-dbcp</groupId>
    8.22 @@ -315,7 +316,8 @@
    8.23              <dependency>
    8.24                  <groupId>org.apache.commons</groupId>
    8.25                  <artifactId>commons-lang3</artifactId>
    8.26 -                <version>3.9</version>
    8.27 +                <!--version>3.9</version-->
    8.28 +                <version>3.7</version>
    8.29              </dependency>
    8.30          	<dependency>
    8.31                  <groupId>commons-fileupload</groupId>
    8.32 @@ -334,7 +336,8 @@
    8.33              <dependency>
    8.34                  <groupId>org.apache.httpcomponents</groupId>
    8.35                  <artifactId>httpclient</artifactId>
    8.36 -                <version>4.5.8</version>
    8.37 +                <!--version>4.5.8</version-->
    8.38 +                <version>4.5.6</version>
    8.39              </dependency>
    8.40  
    8.41  		    <!--+
    8.42 @@ -344,7 +347,8 @@
    8.43              <dependency>
    8.44                  <groupId>com.google.guava</groupId>
    8.45                  <artifactId>guava</artifactId>
    8.46 -                <version>27.1-jre</version>
    8.47 +                <!--version>27.1-jre</version-->
    8.48 +                <version>26.0-jre</version>
    8.49              </dependency>
    8.50  
    8.51  		    <!--+
    8.52 @@ -393,7 +397,8 @@
    8.53              <dependency>
    8.54                  <groupId>joda-time</groupId>
    8.55                  <artifactId>joda-time</artifactId>
    8.56 -                <version>2.10.2</version>
    8.57 +                <!--version>2.12</version-->
    8.58 +                <version>2.10</version>
    8.59              </dependency>
    8.60              <dependency>
    8.61                  <groupId>joda-time</groupId>
    8.62 @@ -435,7 +440,8 @@
    8.63              <dependency>
    8.64                  <groupId>org.jasypt</groupId>
    8.65                  <artifactId>jasypt</artifactId>
    8.66 -                <version>1.9.3</version>
    8.67 +                <!--version>1.9.3</version-->
    8.68 +                <version>1.9.2</version>
    8.69              </dependency>
    8.70  
    8.71  		    <!--+
    8.72 @@ -460,7 +466,8 @@
    8.73              <dependency>
    8.74                  <groupId>org.projectlombok</groupId>
    8.75                  <artifactId>lombok</artifactId>
    8.76 -                <version>1.18.8</version>
    8.77 +                <!--version>1.18.8</version-->
    8.78 +                <version>1.18.2</version>
    8.79              </dependency>
    8.80  
    8.81              <!--+