firethorn

changeset 4282:01988ccd1200

Merged remote changes
author Dave Morris <dmr@roe.ac.uk>
date Wed Oct 09 16:54:28 2019 +0200 (2019-10-09)
parents f3e8908ce635 774e85d540eb
children 2f595232d897
files
line diff
     1.1 --- a/.hgtags	Wed Oct 09 16:53:55 2019 +0200
     1.2 +++ b/.hgtags	Wed Oct 09 16:54:28 2019 +0200
     1.3 @@ -46,3 +46,5 @@
     1.4  da9e0640f31b0f40f440642ad5ce4ebdbc356711 version-2.1.31
     1.5  919418055ef1ff5e3e10e6ad91509251b9991000 version-2.1.32
     1.6  e9d24ed64319a22c950e80e5f8444f760056cb22 version-2.1.33
     1.7 +b3669dab2a603b03e618ec46fe4cad8d1f81a8f6 version-2.1.34
     1.8 +64fc70e71af09d94d73dc692032e6303e184c00b version-2.1.35
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/stv/20190627-Firethorn-Swarm.txt	Wed Oct 09 16:54:28 2019 +0200
     2.3 @@ -0,0 +1,402 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#  
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#
    2.25 +
    2.26 +
    2.27 +## ------------------------------------------------------------------------------------------
    2.28 +## Create Master VM 
    2.29 +## ------------------------------------------------------------------------------------------
    2.30 +
    2.31 +
    2.32 +createvm Cadelicia (master)
    2.33 +..
    2.34 +Domain Cadelicia started
    2.35 +
    2.36 +ssh Stevedore@Cadelicia
    2.37 +
    2.38 +
    2.39 +
    2.40 +## ------------------------------------------------------------------------------------------
    2.41 +## Initialize Swarm with VM's address
    2.42 +## ------------------------------------------------------------------------------------------
    2.43 +
    2.44 +[Stevedore@Cadelicia ~]$ ifconfig
    2.45 +
    2.46 +       ..  192.168.201.11  ..
    2.47 +
    2.48 +
    2.49 +docker swarm init --advertise-addr 192.168.201.11  
    2.50 +
    2.51 +
    2.52 +		Swarm initialized: current node (5zrs2po0x1teeyzigf09mw5vi) is now a manager.
    2.53 +
    2.54 +		To add a worker to this swarm, run the following command:
    2.55 +
    2.56 +		    docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
    2.57 +
    2.58 +		To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    2.59 +
    2.60 +
    2.61 +## ------------------------------------------------------------------------------------------
    2.62 +## Create Spark-net network
    2.63 +## ------------------------------------------------------------------------------------------
    2.64 +## docker network create --driver overlay firethorn-net
    2.65 +
    2.66 +
    2.67 +    ## Fix Firewall issue
    2.68 +    sudo su
    2.69 +	    firewall-cmd --add-port=2377/tcp --permanent
    2.70 +	    firewall-cmd --add-port=2377/tcp --permanent
    2.71 +	    firewall-cmd --add-port=7946/tcp --permanent
    2.72 +	    firewall-cmd --add-port=4789/tcp --permanent
    2.73 +	    firewall-cmd --add-port=4789/udp --permanent
    2.74 +	    iptables -A INPUT -p 50 -j ACCEPT    
    2.75 +	    firewall-cmd --reload
    2.76 +    exit
    2.77 +
    2.78 +
    2.79 +## -------------------------------------------------------------------------------------------
    2.80 +## Add secret function.
    2.81 +## -------------------------------------------------------------------------------------------
    2.82 +
    2.83 +    secrethost='stv@shepseskaf.roe.ac.uk'
    2.84 +    secretfile='${HOME:?}/secret.store.20181212'
    2.85 +
    2.86 +    secret()
    2.87 +    {
    2.88 +        local key=${1:?}
    2.89 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    2.90 +    }
    2.91 +
    2.92 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    2.93 +
    2.94 +
    2.95 +
    2.96 +## -------------------------------------------------------------------------------------------
    2.97 +## Set the target branch and builtag
    2.98 +## -------------------------------------------------------------------------------------------
    2.99 +
   2.100 +
   2.101 +    export buildtag=2.1.33
   2.102 +    export branch=default
   2.103 +
   2.104 +
   2.105 +
   2.106 +## -------------------------------------------------------------------------------------------
   2.107 +## Create our chain.properties file
   2.108 +## -------------------------------------------------------------------------------------------
   2.109 +
   2.110 +cat > "${HOME:?}/chain.properties" << EOF
   2.111 +
   2.112 +metadata=postgres
   2.113 +metauser=$(pwgen 20 1)
   2.114 +metapass=$(pwgen 20 1) 
   2.115 +
   2.116 +userhost=$(secret 'firethorn.user.host')
   2.117 +userdata=$(secret 'firethorn.user.data')
   2.118 +useruser=$(secret 'firethorn.user.user')
   2.119 +userpass=$(secret 'firethorn.user.pass')
   2.120 +usertype=mssql
   2.121 +
   2.122 +datahost=$(secret 'firethorn.data.host')
   2.123 +datadata=$(secret 'firethorn.data.data')
   2.124 +datacatalog=$(secret 'firethorn.data.catalog')
   2.125 +datatype=$(secret 'firethorn.data.type')
   2.126 +datauser=$(secret 'firethorn.data.user')
   2.127 +datapass=$(secret 'firethorn.data.pass')
   2.128 +
   2.129 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   2.130 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   2.131 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   2.132 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   2.133 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   2.134 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   2.135 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   2.136 +
   2.137 +tunneluser=$(secret 'ssh.tunnel.user')
   2.138 +tunnelhost=$(secret 'ssh.tunnel.host')
   2.139 +
   2.140 +admingroup=$(pwgen 20 1)	
   2.141 +adminuser=$(pwgen 20 1)
   2.142 +adminpass=$(pwgen 20 1)
   2.143 +
   2.144 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   2.145 + 
   2.146 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   2.147 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   2.148 +
   2.149 +guestgroup=friends
   2.150 +endpoint=$(secret 'endpoint')
   2.151 +
   2.152 +EOF
   2.153 +
   2.154 +
   2.155 +
   2.156 +## -----------------------------------------------------
   2.157 +## Create our Firethorn properties.
   2.158 +## -----------------------------------------------------
   2.159 +
   2.160 +source "${HOME:?}/chain.properties"
   2.161 +cat > "${HOME:?}/firethorn.properties" << EOF
   2.162 +
   2.163 +firethorn.meta.type=pgsql
   2.164 +firethorn.meta.driver=org.postgresql.Driver
   2.165 +firethorn.meta.host=bethany
   2.166 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   2.167 +firethorn.meta.data=postgres
   2.168 +firethorn.meta.user=${metauser:?}
   2.169 +firethorn.meta.pass=${metapass:?}
   2.170 +
   2.171 +firethorn.user.type=mssql
   2.172 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   2.173 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   2.174 +firethorn.user.host=${userhost:?}
   2.175 +firethorn.user.data=${userdata:?}
   2.176 +firethorn.user.user=${useruser:?}
   2.177 +firethorn.user.pass=${userpass:?}
   2.178 +
   2.179 +firethorn.tapschema.resource.name=${tapschemajdbc}
   2.180 +firethorn.tapschema.database.name=${tapschemadata}
   2.181 +firethorn.tapschema.database.host=${tapschemahost}
   2.182 +firethorn.tapschema.database.port=${tapschemaport}
   2.183 +firethorn.tapschema.database.user=${tapschemauser}
   2.184 +firethorn.tapschema.database.pass=${tapschemapass}
   2.185 +
   2.186 +firethorn.admin.community=${admingroup:?}
   2.187 +firethorn.admin.user.name=${adminuser:?}
   2.188 +firethorn.admin.user.pass=${adminpass:?}
   2.189 +
   2.190 +firethorn.guest.community=${guestgroup:?}
   2.191 +
   2.192 +adql.query.delay.first=${adql_query_delay_first:?}
   2.193 +
   2.194 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   2.195 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   2.196 +	
   2.197 +
   2.198 +EOF
   2.199 +
   2.200 +
   2.201 +
   2.202 +## -------------------------------------------------------------------------------------------
   2.203 +## Create our compose env files.
   2.204 +## -------------------------------------------------------------------------------------------
   2.205 +
   2.206 +source "${HOME:?}/chain.properties"
   2.207 +cat > "${HOME:?}/.env" << EOF
   2.208 +
   2.209 +buildtag=${branch:?}
   2.210 +endpoint=${endpoint:?}
   2.211 +
   2.212 +
   2.213 +metadata=postgres
   2.214 +metauser=${metauser:?}
   2.215 +metapass=${metapass:?}
   2.216 +firethorn.meta.type=pgsql
   2.217 +firethorn.meta.driver=org.postgresql.Driver
   2.218 +firethorn.meta.host=bethany
   2.219 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   2.220 +firethorn.meta.data=postgres
   2.221 +firethorn.meta.user=${metauser:?}
   2.222 +firethorn.meta.pass=${metapass:?}
   2.223 +
   2.224 +
   2.225 +datadata=${datadata:?}
   2.226 +datauser=${datauser:?}
   2.227 +datapass=${datapass:?}
   2.228 +datahost=${datahost:?}
   2.229 +datacatalog=${datacatalog:?}
   2.230 +datatype=mssql
   2.231 +
   2.232 +
   2.233 +tapschemadata=${tapschemadata:?}
   2.234 +tapschemauser=${tapschemauser:?}
   2.235 +tapschemapass=${tapschemapass:?}
   2.236 +tapschemahost=${tapschemahost}
   2.237 +tapschemaport=${tapschemaport:?}
   2.238 +tapschemajdbc=${tapschemajdbc:?}
   2.239 +tapschematype=${tapschematype:?}
   2.240 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   2.241 +firethorn.tapschema.database.name=${tapschemadata:?}
   2.242 +firethorn.tapschema.database.host=${tapschemahost:?}
   2.243 +firethorn.tapschema.database.port=5432
   2.244 +firethorn.tapschema.database.user=${tapschemauser:?}
   2.245 +firethorn.tapschema.database.pass=${tapschemapass:?}
   2.246 +firethorn.tapschema.database.type=pgsql
   2.247 +
   2.248 +userhost=${userhost:?}
   2.249 +userdata=${userdata:?}
   2.250 +useruser=${useruser:?}
   2.251 +userpass=${userpass:?}
   2.252 +usertype=mssql
   2.253 +firethorn.user.type=mssql
   2.254 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   2.255 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   2.256 +firethorn.user.host=${userhost:?}
   2.257 +firethorn.user.data=${userdata:?}
   2.258 +firethorn.user.user=${useruser:?}
   2.259 +firethorn.user.pass=${userpass:?}
   2.260 +
   2.261 +
   2.262 +admingroup=${admingroup:?}
   2.263 +adminuser=${adminuser:?}
   2.264 +adminpass=${adminpass:?}
   2.265 +guestgroup=${guestgroup:?}
   2.266 +firethorn.admin.community=${admingroup:?}
   2.267 +firethorn.admin.user.name=${adminuser:?}
   2.268 +firethorn.admin.user.pass=${adminpass:?}
   2.269 +firethorn.guest.community=${guestgroup:?}
   2.270 +
   2.271 +
   2.272 +adql_query_delay_first=${adql_query_delay_first:?}
   2.273 +adql.query.delay.first=${adql_query_delay_first:?}
   2.274 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   2.275 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   2.276 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   2.277 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   2.278 +
   2.279 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   2.280 +
   2.281 +
   2.282 +EOF
   2.283 +
   2.284 +
   2.285 +source "${HOME:?}/chain.properties"
   2.286 +cat > "${HOME:?}/.tapschema_env" << EOF
   2.287 +
   2.288 +POSTGRES_DB=${tapschemadatabase}
   2.289 +POSTGRES_USER=${tapschemauser}
   2.290 +POSTGRES_PASSWORD=${tapschemapass}
   2.291 +
   2.292 +EOF
   2.293 +
   2.294 +
   2.295 +source "${HOME:?}/chain.properties"
   2.296 +cat > "${HOME:?}/.metadata_env" << EOF
   2.297 +
   2.298 +POSTGRES_DB=${metadata}
   2.299 +POSTGRES_USER=${metauser}
   2.300 +POSTGRES_PASSWORD=${metapass}
   2.301 +         
   2.302 +EOF
   2.303 +
   2.304 +
   2.305 +## -------------------------------------------------------------------------------------------
   2.306 +## Fetch the TAP Deployer
   2.307 +## -------------------------------------------------------------------------------------------
   2.308 +
   2.309 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   2.310 +
   2.311 +
   2.312 +
   2.313 +## ------------------------------------------------------------------------
   2.314 +## Deploy Swarm stack
   2.315 +## ------------------------------------------------------------------------
   2.316 +
   2.317 +docker stack deploy -c docker-compose.yml ft
   2.318 +
   2.319 +
   2.320 +
   2.321 +## ------------------------------------------------------------------------
   2.322 +## To remove all docker services 
   2.323 +## ------------------------------------------------------------------------
   2.324 +
   2.325 +## docker service rm $(docker service ls -q)
   2.326 +
   2.327 +
   2.328 +
   2.329 +## ------------------------------------------------------------------------
   2.330 +## To remove stack of services
   2.331 +## ------------------------------------------------------------------------
   2.332 +
   2.333 +## docker stack rm ft
   2.334 +
   2.335 +
   2.336 +
   2.337 +## ------------------------------------------------------------------------
   2.338 +## Run Docker Compose tap-deployer.yml from firethorn.py
   2.339 +## ------------------------------------------------------------------------
   2.340 +
   2.341 +
   2.342 +source "${HOME:?}/chain.properties"
   2.343 +
   2.344 +docker-compose --file "docker-compose.yml" run firethorn-py
   2.345 +
   2.346 +import firethorn
   2.347 +import firethorn_utils.configurator as configur
   2.348 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   2.349 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   2.350 +configurator =  configur.Configurator(ft)
   2.351 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   2.352 +
   2.353 +..
   2.354 +Importing JDBC Schema: ATLASDR1
   2.355 +Importing JDBC Schema: ATLASDR2
   2.356 +Importing JDBC Schema: ATLASDR3
   2.357 +Importing JDBC Schema: 2MASS
   2.358 +Importing JDBC Schema: 2XMM
   2.359 +Importing JDBC Schema: WISE
   2.360 +Importing JDBC Schema: VIKINGDR3
   2.361 +Importing JDBC Schema: VHSDR1
   2.362 +Importing JDBC Schema: MGC
   2.363 +Importing JDBC Schema: GLIMPSE
   2.364 +Importing JDBC Schema: FIRST
   2.365 +Importing JDBC Schema: DENIS
   2.366 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   2.367 +
   2.368 +..
   2.369 +
   2.370 +## ------------------------------------------------------------------------
   2.371 +## [OPTIONAL] Adding a seconf VM to the Swarm
   2.372 +## ------------------------------------------------------------------------
   2.373 +
   2.374 +
   2.375 +## Create Worker VM 
   2.376 +createvm Erennon (worker)
   2.377 +Domain Erennon started
   2.378 +
   2.379 +ssh Stevedore@Erennon
   2.380 +
   2.381 +     docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
   2.382 +      ...
   2.383 +      This node joined a swarm as a worker.
   2.384 +      ...
   2.385 +
   2.386 +    
   2.387 +     ## Fix Firewall issue
   2.388 +     sudo su
   2.389 +     firewall-cmd --add-port=2377/tcp --permanent
   2.390 +     firewall-cmd --add-port=2377/tcp --permanent
   2.391 +     firewall-cmd --add-port=7946/tcp --permanent
   2.392 +     firewall-cmd --add-port=4789/udp --permanent
   2.393 +     firewall-cmd --add-port=4789/tcp --permanent
   2.394 +     iptables -A INPUT -p 50 -j ACCEPT    
   2.395 +     firewall-cmd --reload
   2.396 +     exit
   2.397 +
   2.398 +
   2.399 +## ------------------------------------------------------------------------
   2.400 +##  Scaling: If we were to scale the cluster, we Could do so as:
   2.401 +## ------------------------------------------------------------------------
   2.402 +docker service scale ft_gillian=4
   2.403 +docker service scale ft_jarmila=4
   2.404 +
   2.405 +## But this will not do anything at the moment
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/doc/notes/stv/20190718-Firethorn-Swarm.txt	Wed Oct 09 16:54:28 2019 +0200
     3.3 @@ -0,0 +1,536 @@
     3.4 +#
     3.5 +# <meta:header>
     3.6 +#   <meta:licence>
     3.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     3.8 +#
     3.9 +#     This information is free software: you can redistribute it and/or modify
    3.10 +#     it under the terms of the GNU General Public License as published by
    3.11 +#     the Free Software Foundation, either version 3 of the License, or
    3.12 +#     (at your option) any later version.
    3.13 +#
    3.14 +#     This information is distributed in the hope that it will be useful,
    3.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    3.17 +#     GNU General Public License for more details.
    3.18 +#  
    3.19 +#     You should have received a copy of the GNU General Public License
    3.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    3.21 +#   </meta:licence>
    3.22 +# </meta:header>
    3.23 +#
    3.24 +#
    3.25 +
    3.26 +
    3.27 +## ------------------------------------------------------------------------------------------
    3.28 +## Create Master VM 
    3.29 +## ------------------------------------------------------------------------------------------
    3.30 +
    3.31 +
    3.32 +createvm Froeseth (master)
    3.33 +..
    3.34 +Domain Froeseth started
    3.35 +
    3.36 +ssh Stevedore@Froeseth
    3.37 +
    3.38 +
    3.39 +
    3.40 +## ------------------------------------------------------------------------------------------
    3.41 +## Initialize Swarm with VM's address
    3.42 +## ------------------------------------------------------------------------------------------
    3.43 +
    3.44 +[Stevedore@Froeseth ~]$ ifconfig
    3.45 +
    3.46 +       ..  192.168.201.12  ..
    3.47 +
    3.48 +
    3.49 +docker swarm init --advertise-addr 192.168.201.12 
    3.50 +
    3.51 +
    3.52 +		Swarm initialized: current node (5zrs2po0x1teeyzigf09mw5vi) is now a manager.
    3.53 +
    3.54 +		To add a worker to this swarm, run the following command:
    3.55 +
    3.56 +		    docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
    3.57 +
    3.58 +		To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    3.59 +
    3.60 +
    3.61 +## ------------------------------------------------------------------------------------------
    3.62 +## Create Spark-net network
    3.63 +## ------------------------------------------------------------------------------------------
    3.64 +## docker network create --driver overlay firethorn-net
    3.65 +
    3.66 +
    3.67 +    ## Fix Firewall issue
    3.68 +    sudo su
    3.69 +	    firewall-cmd --add-port=2377/tcp --permanent
    3.70 +	    firewall-cmd --add-port=2377/tcp --permanent
    3.71 +	    firewall-cmd --add-port=7946/tcp --permanent
    3.72 +	    firewall-cmd --add-port=4789/tcp --permanent
    3.73 +	    firewall-cmd --add-port=4789/udp --permanent
    3.74 +	    iptables -A INPUT -p 50 -j ACCEPT    
    3.75 +	    firewall-cmd --reload
    3.76 +    exit
    3.77 +
    3.78 +
    3.79 +## -------------------------------------------------------------------------------------------
    3.80 +## Add secret function.
    3.81 +## -------------------------------------------------------------------------------------------
    3.82 +
    3.83 +    secrethost='stv@shepseskaf.roe.ac.uk'
    3.84 +    secretfile='${HOME:?}/secret.store...'
    3.85 +
    3.86 +    secret()
    3.87 +    {
    3.88 +        local key=${1:?}
    3.89 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    3.90 +    }
    3.91 +
    3.92 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    3.93 +
    3.94 +
    3.95 +
    3.96 +## -------------------------------------------------------------------------------------------
    3.97 +## Set the target branch and builtag
    3.98 +## -------------------------------------------------------------------------------------------
    3.99 +
   3.100 +
   3.101 +    export buildtag=2.1.33-stv-tap-swarm
   3.102 +    export branch=2.1.33-stv-tap-swarm
   3.103 +
   3.104 +
   3.105 +
   3.106 +## -------------------------------------------------------------------------------------------
   3.107 +## Create our chain.properties file
   3.108 +## -------------------------------------------------------------------------------------------
   3.109 +
   3.110 +cat > "${HOME:?}/chain.properties" << EOF
   3.111 +
   3.112 +metadata=postgres
   3.113 +metauser=$(pwgen 20 1)
   3.114 +metapass=$(pwgen 20 1) 
   3.115 +
   3.116 +userhost=$(secret 'firethorn.user.host')
   3.117 +userdata=$(secret 'firethorn.user.data')
   3.118 +useruser=$(secret 'firethorn.user.user')
   3.119 +userpass=$(secret 'firethorn.user.pass')
   3.120 +usertype=mssql
   3.121 +
   3.122 +datahost=$(secret 'firethorn.data.host')
   3.123 +datadata=$(secret 'firethorn.data.data')
   3.124 +datacatalog=$(secret 'firethorn.data.catalog')
   3.125 +datatype=$(secret 'firethorn.data.type')
   3.126 +datauser=$(secret 'firethorn.data.user')
   3.127 +datapass=$(secret 'firethorn.data.pass')
   3.128 +
   3.129 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   3.130 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   3.131 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   3.132 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   3.133 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   3.134 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   3.135 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   3.136 +
   3.137 +tunneluser=$(secret 'ssh.tunnel.user')
   3.138 +tunnelhost=$(secret 'ssh.tunnel.host')
   3.139 +
   3.140 +admingroup=$(pwgen 20 1)	
   3.141 +adminuser=$(pwgen 20 1)
   3.142 +adminpass=$(pwgen 20 1)
   3.143 +
   3.144 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   3.145 + 
   3.146 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   3.147 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   3.148 +
   3.149 +guestgroup=friends
   3.150 +endpoint=$(secret 'endpoint')
   3.151 +
   3.152 +firethornhost=gillian
   3.153 +
   3.154 +EOF
   3.155 +
   3.156 +
   3.157 +
   3.158 +## -----------------------------------------------------
   3.159 +## Create our Firethorn properties.
   3.160 +## -----------------------------------------------------
   3.161 +
   3.162 +source "${HOME:?}/chain.properties"
   3.163 +cat > "${HOME:?}/firethorn.properties" << EOF
   3.164 +
   3.165 +firethorn.meta.type=pgsql
   3.166 +firethorn.meta.driver=org.postgresql.Driver
   3.167 +firethorn.meta.host=bethany
   3.168 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   3.169 +firethorn.meta.data=postgres
   3.170 +firethorn.meta.user=${metauser:?}
   3.171 +firethorn.meta.pass=${metapass:?}
   3.172 +
   3.173 +firethorn.user.type=mssql
   3.174 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   3.175 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   3.176 +firethorn.user.host=${userhost:?}
   3.177 +firethorn.user.data=${userdata:?}
   3.178 +firethorn.user.user=${useruser:?}
   3.179 +firethorn.user.pass=${userpass:?}
   3.180 +
   3.181 +firethorn.tapschema.resource.name=${tapschemajdbc}
   3.182 +firethorn.tapschema.database.name=${tapschemadata}
   3.183 +firethorn.tapschema.database.host=${tapschemahost}
   3.184 +firethorn.tapschema.database.port=${tapschemaport}
   3.185 +firethorn.tapschema.database.user=${tapschemauser}
   3.186 +firethorn.tapschema.database.pass=${tapschemapass}
   3.187 +
   3.188 +firethorn.admin.community=${admingroup:?}
   3.189 +firethorn.admin.user.name=${adminuser:?}
   3.190 +firethorn.admin.user.pass=${adminpass:?}
   3.191 +
   3.192 +firethorn.guest.community=${guestgroup:?}
   3.193 +
   3.194 +adql.query.delay.first=${adql_query_delay_first:?}
   3.195 +
   3.196 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   3.197 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   3.198 +	
   3.199 +
   3.200 +EOF
   3.201 +
   3.202 +
   3.203 +
   3.204 +## -------------------------------------------------------------------------------------------
   3.205 +## Create our compose env files.
   3.206 +## -------------------------------------------------------------------------------------------
   3.207 +
   3.208 +source "${HOME:?}/chain.properties"
   3.209 +cat > "${HOME:?}/.env" << EOF
   3.210 +
   3.211 +buildtag=${branch:?}
   3.212 +endpoint=${endpoint:?}
   3.213 +
   3.214 +
   3.215 +metadata=postgres
   3.216 +metauser=${metauser:?}
   3.217 +metapass=${metapass:?}
   3.218 +firethorn.meta.type=pgsql
   3.219 +firethorn.meta.driver=org.postgresql.Driver
   3.220 +firethorn.meta.host=bethany
   3.221 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   3.222 +firethorn.meta.data=postgres
   3.223 +firethorn.meta.user=${metauser:?}
   3.224 +firethorn.meta.pass=${metapass:?}
   3.225 +
   3.226 +
   3.227 +datadata=${datadata:?}
   3.228 +datauser=${datauser:?}
   3.229 +datapass=${datapass:?}
   3.230 +datahost=${datahost:?}
   3.231 +datacatalog=${datacatalog:?}
   3.232 +datatype=mssql
   3.233 +
   3.234 +
   3.235 +tapschemadata=${tapschemadata:?}
   3.236 +tapschemauser=${tapschemauser:?}
   3.237 +tapschemapass=${tapschemapass:?}
   3.238 +tapschemahost=${tapschemahost}
   3.239 +tapschemaport=${tapschemaport:?}
   3.240 +tapschemajdbc=${tapschemajdbc:?}
   3.241 +tapschematype=${tapschematype:?}
   3.242 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   3.243 +firethorn.tapschema.database.name=${tapschemadata:?}
   3.244 +firethorn.tapschema.database.host=${tapschemahost:?}
   3.245 +firethorn.tapschema.database.port=5432
   3.246 +firethorn.tapschema.database.user=${tapschemauser:?}
   3.247 +firethorn.tapschema.database.pass=${tapschemapass:?}
   3.248 +firethorn.tapschema.database.type=pgsql
   3.249 +
   3.250 +userhost=${userhost:?}
   3.251 +userdata=${userdata:?}
   3.252 +useruser=${useruser:?}
   3.253 +userpass=${userpass:?}
   3.254 +usertype=mssql
   3.255 +firethorn.user.type=mssql
   3.256 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   3.257 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   3.258 +firethorn.user.host=${userhost:?}
   3.259 +firethorn.user.data=${userdata:?}
   3.260 +firethorn.user.user=${useruser:?}
   3.261 +firethorn.user.pass=${userpass:?}
   3.262 +
   3.263 +
   3.264 +admingroup=${admingroup:?}
   3.265 +adminuser=${adminuser:?}
   3.266 +adminpass=${adminpass:?}
   3.267 +guestgroup=${guestgroup:?}
   3.268 +firethorn.admin.community=${admingroup:?}
   3.269 +firethorn.admin.user.name=${adminuser:?}
   3.270 +firethorn.admin.user.pass=${adminpass:?}
   3.271 +firethorn.guest.community=${guestgroup:?}
   3.272 +
   3.273 +
   3.274 +adql_query_delay_first=${adql_query_delay_first:?}
   3.275 +adql.query.delay.first=${adql_query_delay_first:?}
   3.276 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   3.277 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   3.278 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   3.279 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   3.280 +
   3.281 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   3.282 +
   3.283 +firethornhost=gillian
   3.284 +
   3.285 +EOF
   3.286 +
   3.287 +
   3.288 +source "${HOME:?}/chain.properties"
   3.289 +cat > "${HOME:?}/.tapschema_env" << EOF
   3.290 +
   3.291 +POSTGRES_DB=${tapschemadatabase}
   3.292 +POSTGRES_USER=${tapschemauser}
   3.293 +POSTGRES_PASSWORD=${tapschemapass}
   3.294 +
   3.295 +EOF
   3.296 +
   3.297 +
   3.298 +source "${HOME:?}/chain.properties"
   3.299 +cat > "${HOME:?}/.metadata_env" << EOF
   3.300 +
   3.301 +POSTGRES_DB=${metadata}
   3.302 +POSTGRES_USER=${metauser}
   3.303 +POSTGRES_PASSWORD=${metapass}
   3.304 +         
   3.305 +EOF
   3.306 +
   3.307 +
   3.308 +## -------------------------------------------------------------------------------------------
   3.309 +## Fetch the TAP Deployer
   3.310 +## -------------------------------------------------------------------------------------------
   3.311 +
   3.312 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   3.313 +
   3.314 +
   3.315 +
   3.316 +## -------------------------------------------------------------------------------------------
   3.317 +## Build from Firethorn Source (2.1.33-stv-tap-swarm branch)
   3.318 +## -------------------------------------------------------------------------------------------
   3.319 +
   3.320 +sudo yum -y install maven
   3.321 +
   3.322 +hg clone http://wfau.metagrid.co.uk/code/firethorn hg
   3.323 +
   3.324 +
   3.325 +
   3.326 +cat > "${HOME:?}/firethorn.settings" << EOF
   3.327 +FIRETHORN_NAME=hg
   3.328 +FIRETHORN_BASE=/home/Stevedore/hg
   3.329 +FIRETHORN_CODE=/home/Stevedore/hg
   3.330 +FIRETHORN_CONF=/home/Stevedore/hg
   3.331 +EOF
   3.332 +
   3.333 +
   3.334 +
   3.335 +pushd ${HOME:?}/hg
   3.336 +
   3.337 +    hg update -C 2.1.33-stv-tap-swarm
   3.338 +
   3.339 +popd
   3.340 +
   3.341 +
   3.342 +
   3.343 +## -----------------------------------------------------
   3.344 +## Build our Docker images.
   3.345 +## -----------------------------------------------------
   3.346 +
   3.347 +    source "${HOME:?}/firethorn.settings"
   3.348 +    pushd "${FIRETHORN_CODE:?}"
   3.349 +
   3.350 +        export buildtag=$(hg branch)
   3.351 +
   3.352 +        docker-compose \
   3.353 +            --file docker/compose/images.yml \
   3.354 +            build
   3.355 +
   3.356 +    popd
   3.357 +
   3.358 +
   3.359 +
   3.360 +
   3.361 +##---------------------------------------------------------------------
   3.362 +## Compile our Java code.
   3.363 +##---------------------------------------------------------------------
   3.364 +
   3.365 +    source "${HOME:?}/firethorn.settings"
   3.366 +    pushd "${FIRETHORN_CODE:?}"
   3.367 +
   3.368 +        mvn clean install
   3.369 +
   3.370 +    popd
   3.371 +
   3.372 +
   3.373 +##---------------------------------------------------------------------
   3.374 +## Build our Java containers.
   3.375 +##---------------------------------------------------------------------
   3.376 +
   3.377 +    source "${HOME:?}/firethorn.settings"
   3.378 +    pushd "${FIRETHORN_CODE:?}"
   3.379 +
   3.380 +        export buildtag=$(hg branch)
   3.381 +        pushd firethorn-ogsadai/webapp
   3.382 +            mvn docker:package
   3.383 +        popd
   3.384 +
   3.385 +        export buildtag=$(hg branch)
   3.386 +        pushd firethorn-webapp
   3.387 +            mvn docker:package
   3.388 +        popd
   3.389 +
   3.390 +    popd
   3.391 +
   3.392 +
   3.393 +
   3.394 +## ------------------------------------------------------------------------
   3.395 +## Deploy Swarm stack
   3.396 +## ------------------------------------------------------------------------
   3.397 +
   3.398 +docker stack deploy -c docker-compose.yml ft
   3.399 +        .. 
   3.400 +	Creating service ft_bethany
   3.401 +	Creating service ft_carolina
   3.402 +	Creating service ft_jarmila
   3.403 +	Creating service ft_gillian
   3.404 +	Creating service ft_firethorn-py
   3.405 +     
   3.406 +
   3.407 +
   3.408 +[Stevedore@Froeseth ~]$ docker service ls
   3.409 +ID                  NAME                MODE                REPLICAS            IMAGE                                         PORTS
   3.410 +su0iy3uspkya        ft_bethany          replicated          1/1                 firethorn/postgres:2.1.33-stv-tap-swarm       
   3.411 +rve9iiumjoj4        ft_carolina         replicated          1/1                 firethorn/postgres:2.1.33-stv-tap-swarm       
   3.412 +7fswcw9btvct        ft_firethorn-py     replicated          0/1                 firethorn/firethorn-py:2.1.33-stv-tap-swarm   
   3.413 +l52nhtsaj5qy        ft_gillian          replicated          1/1                 firethorn/firethorn:2.1.33-stv-tap-swarm      *:8080->8080/tcp
   3.414 +2ubzla3luwe9        ft_jarmila          replicated          1/1                 firethorn/ogsadai:2.1.33-stv-tap-swarm 
   3.415 +
   3.416 +## ------------------------------------------------------------------------
   3.417 +## To remove all docker services 
   3.418 +## ------------------------------------------------------------------------
   3.419 +
   3.420 +## docker service rm $(docker service ls -q)
   3.421 +
   3.422 +
   3.423 +
   3.424 +## ------------------------------------------------------------------------
   3.425 +## Create TAP service with firethorn-utils
   3.426 +## ------------------------------------------------------------------------
   3.427 +
   3.428 +docker start ft_firethorn-py.1.txmiebaz5mohrlqo632ngb0x6
   3.429 +docker exec -it ft_firethorn-py.1.txmiebaz5mohrlqo632ngb0x6 bash
   3.430 +
   3.431 +root@firethorn-py:/home# python3
   3.432 +
   3.433 +
   3.434 +import firethorn
   3.435 +import firethorn_utils.configurator as configur
   3.436 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   3.437 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   3.438 +configurator =  configur.Configurator(ft)
   3.439 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   3.440 +
   3.441 +Importing JDBC Schema: ATLASDR1
   3.442 +Importing JDBC Schema: ATLASDR2
   3.443 +Importing JDBC Schema: ATLASDR3
   3.444 +Importing JDBC Schema: 2MASS
   3.445 +Importing JDBC Schema: 2XMM
   3.446 +Importing JDBC Schema: WISE
   3.447 +Importing JDBC Schema: VIKINGDR3
   3.448 +Importing JDBC Schema: VHSDR1
   3.449 +Importing JDBC Schema: MGC
   3.450 +Importing JDBC Schema: GLIMPSE
   3.451 +Importing JDBC Schema: FIRST
   3.452 +Importing JDBC Schema: DENIS
   3.453 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   3.454 +
   3.455 +
   3.456 +
   3.457 +	## ------------------------------------------------------------------------
   3.458 +## Update Apache Proxy 
   3.459 +## ------------------------------------------------------------------------
   3.460 +
   3.461 +[Stevedore@Gworewia ~]$ docker exec -it apache bash
   3.462 +
   3.463 +nano /etc/httpd/conf.d/tap.metagrid.xyz.conf
   3.464 +
   3.465 +    ProxyPassMatch    ^/osa\/(.*)$  http://froeseth:8080/firethorn/tap/54/$1
   3.466 +    ProxyPassReverse  ^/osa\/(.*)$  http://froeseth:8080/firethorn/tap/54/$1
   3.467 +
   3.468 +[Stevedore@Gworewia ~]$ docker kill --signal 'USR1' 'apache'
   3.469 +
   3.470 +
   3.471 +
   3.472 +## ------------------------------------------------------------------------
   3.473 +## Setup Local tunnel to Gworewia (Proxy)
   3.474 +## ------------------------------------------------------------------------
   3.475 +
   3.476 +## From local machine
   3.477 +ssh -L '*:8081:Gworewia:80' Stevedore@Gworewia 
   3.478 +
   3.479 +
   3.480 +
   3.481 +
   3.482 +## From local machine
   3.483 +
   3.484 +
   3.485 +cat parallel.py
   3.486 +
   3.487 +
   3.488 +import os                                                                       
   3.489 +from multiprocessing import Pool                                                
   3.490 +
   3.491 +process_list = range(100)
   3.492 +	
   3.493 +def run_process(process_counter):                                                             
   3.494 +    print ("Starting RUN # " + str(process_counter))
   3.495 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://localhost:8081/firethorn -r=54 -u= -p= -g= -v=True -m=sync')
   3.496 +    print ("Ended RUN # " + str(process_counter))
   3.497 +
   3.498 +
   3.499 +pool = Pool(processes=20)                                                        
   3.500 +pool.map(run_process, process_list)   
   3.501 +
   3.502 +
   3.503 +
   3.504 +.. 
   3.505 +
   3.506 +
   3.507 +
   3.508 +python3 parallel.py
   3.509 +
   3.510 +
   3.511 +..
   3.512 +
   3.513 +## After running this for about an hour, queries start failing, and firethorn stops responding
   3.514 +
   3.515 +[Stevedore@Froeseth ~]$ docker ps
   3.516 +CONTAINER ID        IMAGE                                      COMMAND                  CREATED             STATUS                             PORTS               NAMES
   3.517 +7619b2240e8c        firethorn/ogsadai:2.1.33-stv-tap-swarm     "/bin/sh -c '/var/lo…"   52 seconds ago      Up 47 seconds (health: starting)   8080/tcp            ft_jarmila.1.wlbqvl76nso5rqfhjv3bz45sj
   3.518 +be6c8a6c033f        firethorn/firethorn:2.1.33-stv-tap-swarm   "/bin/sh -c '/var/lo…"   4 minutes ago       Up 3 minutes (unhealthy)           8080/tcp            ft_gillian.1.agyapxmtenrr8harqboqheu7c
   3.519 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours                        5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
   3.520 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours                        5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
   3.521 +59b0ce098117        firethorn/firethorn-py:latest              "python3"                23 hours ago        Up 23 hours                                            ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
   3.522 +
   3.523 +
   3.524 +
   3.525 +
   3.526 +## Healthchecker is triggered and both Firethorn and Ogsadai recover with a "healthy" status
   3.527 +
   3.528 +
   3.529 +[Stevedore@Froeseth ~]$ docker ps
   3.530 +CONTAINER ID        IMAGE                                      COMMAND                  CREATED             STATUS                   PORTS               NAMES
   3.531 +5406b10d7fd2        firethorn/firethorn:2.1.33-stv-tap-swarm   "/bin/sh -c '/var/lo…"   3 minutes ago       Up 3 minutes (healthy)   8080/tcp            ft_gillian.1.zfo4zd8q2kgisk49d73xki4qj
   3.532 +7619b2240e8c        firethorn/ogsadai:2.1.33-stv-tap-swarm     "/bin/sh -c '/var/lo…"   4 minutes ago       Up 4 minutes (healthy)   8080/tcp            ft_jarmila.1.wlbqvl76nso5rqfhjv3bz45sj
   3.533 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours              5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
   3.534 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours              5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
   3.535 +59b0ce098117        firethorn/firethorn-py:latest              "python3"                23 hours ago        Up 23 hours                                  ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
   3.536 +
   3.537 +
   3.538 +
   3.539 +## Queries start working again..
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/doc/notes/stv/20190720-Firethorn-Swarm-Deploy.txt	Wed Oct 09 16:54:28 2019 +0200
     4.3 @@ -0,0 +1,648 @@
     4.4 +#
     4.5 +# <meta:header>
     4.6 +#   <meta:licence>
     4.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     4.8 +#
     4.9 +#     This information is free software: you can redistribute it and/or modify
    4.10 +#     it under the terms of the GNU General Public License as published by
    4.11 +#     the Free Software Foundation, either version 3 of the License, or
    4.12 +#     (at your option) any later version.
    4.13 +#
    4.14 +#     This information is distributed in the hope that it will be useful,
    4.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    4.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    4.17 +#     GNU General Public License for more details.
    4.18 +#  
    4.19 +#     You should have received a copy of the GNU General Public License
    4.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    4.21 +#   </meta:licence>
    4.22 +# </meta:header>
    4.23 +#
    4.24 +#
    4.25 +
    4.26 +############################################################################
    4.27 +##### Notes on Deploying WFAU TAP Services on trop02 with Docker Swarm #####
    4.28 +############################################################################
    4.29 +
    4.30 +## Aim of 2.1.34 version is an autorecovering TAP Service, using Swarm, addressing a Swarm Networking issue
    4.31 +## OGSADAI cannot get the original IP address of Firethorn in docker swarm mode 
    4.32 +## https://github.com/moby/moby/issues/25526
    4.33 +
    4.34 +stv@trop02:~$ createvm
    4.35 +
    4.36 +
    4.37 +	## INFO : Node name [Lothigometh]
    4.38 +	## INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    4.39 +	## INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    4.40 +	## INFO : Disc name [Lothigometh.qcow]
    4.41 +	## INFO : Disc size [32GiB]
    4.42 +
    4.43 +
    4.44 +
    4.45 +
    4.46 +ssh Stevedore@Lothigometh
    4.47 +
    4.48 +
    4.49 +
    4.50 +## ------------------------------------------------------------------------------------------
    4.51 +## Initialize Swarm with VM's address
    4.52 +## ------------------------------------------------------------------------------------------
    4.53 +
    4.54 +[Stevedore@Lothigometh ~]$ ifconfig
    4.55 +
    4.56 +       ## ..  192.168.202.9  ..
    4.57 +
    4.58 +
    4.59 +
    4.60 +docker swarm init --advertise-addr 192.168.202.9
    4.61 +
    4.62 +
    4.63 +	# To add a worker to this swarm, run the following command:
    4.64 +
    4.65 +	#     docker swarm join --token SWMTKN-1-5e6vdte3jggnih4t2fu5vrk4o0p2twg5mcjy7bi29s8lrv1uxs-6b09cgappan7fv9myme1pi39w 192.168.202.9:2377
    4.66 +
    4.67 +	# To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    4.68 +
    4.69 +
    4.70 +
    4.71 +## ------------------------------------------------------------------------------------------
    4.72 +## Configure Firewall for Docker Swarm
    4.73 +## https://www.digitalocean.com/community/tutorials/how-to-configure-the-linux-firewall-for-docker-swarm-on-centos-7
    4.74 +## ------------------------------------------------------------------------------------------
    4.75 +
    4.76 +
    4.77 +
    4.78 +    ## Fix Firewall issue
    4.79 +    sudo su
    4.80 +	    firewall-cmd --add-port=2377/tcp --permanent
    4.81 +	    firewall-cmd --add-port=2377/tcp --permanent
    4.82 +	    firewall-cmd --add-port=7946/tcp --permanent
    4.83 +	    firewall-cmd --add-port=4789/tcp --permanent
    4.84 +	    firewall-cmd --add-port=4789/udp --permanent
    4.85 +	    firewall-cmd --reload
    4.86 +    exit
    4.87 +
    4.88 +
    4.89 +## ------------------------------------------------------------------------------------------
    4.90 +## Set the target branch and builtag
    4.91 +## ------------------------------------------------------------------------------------------
    4.92 +
    4.93 +
    4.94 +    export buildtag=2.1.34
    4.95 +    export branch=default
    4.96 +
    4.97 +
    4.98 +
    4.99 +## ------------------------------------------------------------------------------------------
   4.100 +## Add secret function.
   4.101 +## ------------------------------------------------------------------------------------------
   4.102 +
   4.103 +    secrethost='stv@shepseskaf.roe.ac.uk'
   4.104 +    secretfile='${HOME:?}/secret.store.20181212'
   4.105 +
   4.106 +    secret()
   4.107 +    {
   4.108 +        local key=${1:?}
   4.109 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
   4.110 +    }
   4.111 +
   4.112 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
   4.113 +
   4.114 +
   4.115 +# ssh: connect to host shepseskaf.roe.ac.uk port 22: No route to host
   4.116 +# Shepseskaf is not accessible, looks like its down (todo: email Mark to see what is going on)
   4.117 +
   4.118 +
   4.119 +# For now, use properties from the last deploy 
   4.120 +# Stevedore@Araybwyn (/home/Stevedore/chain.properties)
   4.121 +# The example below is what is needed when the above secret function can access the server where the secrets are stored
   4.122 +
   4.123 +
   4.124 +
   4.125 +
   4.126 +## -------------------------------------------------------------------------------------------
   4.127 +## Create our chain.properties file
   4.128 +## -------------------------------------------------------------------------------------------
   4.129 +
   4.130 +cat > "${HOME:?}/chain.properties" << EOF
   4.131 +
   4.132 +metadata=postgres
   4.133 +metauser=$(pwgen 20 1)
   4.134 +metapass=$(pwgen 20 1) 
   4.135 +
   4.136 +userhost=$(secret 'firethorn.user.host')
   4.137 +userdata=$(secret 'firethorn.user.data')
   4.138 +useruser=$(secret 'firethorn.user.user')
   4.139 +userpass=$(secret 'firethorn.user.pass')
   4.140 +usertype=mssql
   4.141 +
   4.142 +datahost=$(secret 'firethorn.data.host')
   4.143 +datadata=$(secret 'firethorn.data.data')
   4.144 +datacatalog=$(secret 'firethorn.data.catalog')
   4.145 +datatype=$(secret 'firethorn.data.type')
   4.146 +datauser=$(secret 'firethorn.data.user')
   4.147 +datapass=$(secret 'firethorn.data.pass')
   4.148 +
   4.149 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   4.150 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   4.151 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   4.152 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   4.153 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   4.154 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   4.155 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   4.156 +
   4.157 +tunneluser=$(secret 'ssh.tunnel.user')
   4.158 +tunnelhost=$(secret 'ssh.tunnel.host')
   4.159 +
   4.160 +admingroup=$(pwgen 20 1)	
   4.161 +adminuser=$(pwgen 20 1)
   4.162 +adminpass=$(pwgen 20 1)
   4.163 +
   4.164 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   4.165 + 
   4.166 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   4.167 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   4.168 +
   4.169 +guestgroup=friends
   4.170 +endpoint=$(secret 'endpoint')
   4.171 +
   4.172 +firethornhost=gillian
   4.173 +
   4.174 +EOF
   4.175 +
   4.176 +
   4.177 +
   4.178 +## ------------------------------------------------------------------------------------------
   4.179 +## Create our Firethorn properties.
   4.180 +## ------------------------------------------------------------------------------------------
   4.181 +
   4.182 +source "${HOME:?}/chain.properties"
   4.183 +cat > "${HOME:?}/firethorn.properties" << EOF
   4.184 +
   4.185 +firethorn.meta.type=pgsql
   4.186 +firethorn.meta.driver=org.postgresql.Driver
   4.187 +firethorn.meta.host=bethany
   4.188 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   4.189 +firethorn.meta.data=postgres
   4.190 +firethorn.meta.user=${metauser:?}
   4.191 +firethorn.meta.pass=${metapass:?}
   4.192 +
   4.193 +firethorn.user.type=mssql
   4.194 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   4.195 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   4.196 +firethorn.user.host=${userhost:?}
   4.197 +firethorn.user.data=${userdata:?}
   4.198 +firethorn.user.user=${useruser:?}
   4.199 +firethorn.user.pass=${userpass:?}
   4.200 +
   4.201 +firethorn.tapschema.resource.name=${tapschemajdbc}
   4.202 +firethorn.tapschema.database.name=${tapschemadata}
   4.203 +firethorn.tapschema.database.host=${tapschemahost}
   4.204 +firethorn.tapschema.database.port=${tapschemaport}
   4.205 +firethorn.tapschema.database.user=${tapschemauser}
   4.206 +firethorn.tapschema.database.pass=${tapschemapass}
   4.207 +
   4.208 +firethorn.admin.community=${admingroup:?}
   4.209 +firethorn.admin.user.name=${adminuser:?}
   4.210 +firethorn.admin.user.pass=${adminpass:?}
   4.211 +
   4.212 +firethorn.guest.community=${guestgroup:?}
   4.213 +
   4.214 +adql.query.delay.first=${adql_query_delay_first:?}
   4.215 +
   4.216 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   4.217 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   4.218 +	
   4.219 +
   4.220 +EOF
   4.221 +
   4.222 +
   4.223 +
   4.224 +## ------------------------------------------------------------------------------------------
   4.225 +## Create our compose env files.
   4.226 +## ------------------------------------------------------------------------------------------
   4.227 +
   4.228 +source "${HOME:?}/chain.properties"
   4.229 +cat > "${HOME:?}/.env" << EOF
   4.230 +
   4.231 +buildtag=${branch:?}
   4.232 +endpoint=${endpoint:?}
   4.233 +
   4.234 +
   4.235 +metadata=postgres
   4.236 +metauser=${metauser:?}
   4.237 +metapass=${metapass:?}
   4.238 +firethorn.meta.type=pgsql
   4.239 +firethorn.meta.driver=org.postgresql.Driver
   4.240 +firethorn.meta.host=bethany
   4.241 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   4.242 +firethorn.meta.data=postgres
   4.243 +firethorn.meta.user=${metauser:?}
   4.244 +firethorn.meta.pass=${metapass:?}
   4.245 +
   4.246 +
   4.247 +datadata=${datadata:?}
   4.248 +datauser=${datauser:?}
   4.249 +datapass=${datapass:?}
   4.250 +datahost=${datahost:?}
   4.251 +datacatalog=${datacatalog:?}
   4.252 +datatype=mssql
   4.253 +
   4.254 +
   4.255 +tapschemadata=${tapschemadata:?}
   4.256 +tapschemauser=${tapschemauser:?}
   4.257 +tapschemapass=${tapschemapass:?}
   4.258 +tapschemahost=${tapschemahost}
   4.259 +tapschemaport=${tapschemaport:?}
   4.260 +tapschemajdbc=${tapschemajdbc:?}
   4.261 +tapschematype=${tapschematype:?}
   4.262 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   4.263 +firethorn.tapschema.database.name=${tapschemadata:?}
   4.264 +firethorn.tapschema.database.host=${tapschemahost:?}
   4.265 +firethorn.tapschema.database.port=5432
   4.266 +firethorn.tapschema.database.user=${tapschemauser:?}
   4.267 +firethorn.tapschema.database.pass=${tapschemapass:?}
   4.268 +firethorn.tapschema.database.type=pgsql
   4.269 +
   4.270 +userhost=${userhost:?}
   4.271 +userdata=${userdata:?}
   4.272 +useruser=${useruser:?}
   4.273 +userpass=${userpass:?}
   4.274 +usertype=mssql
   4.275 +firethorn.user.type=mssql
   4.276 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   4.277 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   4.278 +firethorn.user.host=${userhost:?}
   4.279 +firethorn.user.data=${userdata:?}
   4.280 +firethorn.user.user=${useruser:?}
   4.281 +firethorn.user.pass=${userpass:?}
   4.282 +
   4.283 +
   4.284 +admingroup=${admingroup:?}
   4.285 +adminuser=${adminuser:?}
   4.286 +adminpass=${adminpass:?}
   4.287 +guestgroup=${guestgroup:?}
   4.288 +firethorn.admin.community=${admingroup:?}
   4.289 +firethorn.admin.user.name=${adminuser:?}
   4.290 +firethorn.admin.user.pass=${adminpass:?}
   4.291 +firethorn.guest.community=${guestgroup:?}
   4.292 +
   4.293 +
   4.294 +adql_query_delay_first=${adql_query_delay_first:?}
   4.295 +adql.query.delay.first=${adql_query_delay_first:?}
   4.296 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   4.297 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   4.298 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   4.299 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   4.300 +
   4.301 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   4.302 +
   4.303 +firethornhost=gillian
   4.304 +
   4.305 +EOF
   4.306 +
   4.307 +
   4.308 +source "${HOME:?}/chain.properties"
   4.309 +cat > "${HOME:?}/.tapschema_env" << EOF
   4.310 +
   4.311 +POSTGRES_DB=${tapschemadatabase}
   4.312 +POSTGRES_USER=${tapschemauser}
   4.313 +POSTGRES_PASSWORD=${tapschemapass}
   4.314 +
   4.315 +EOF
   4.316 +
   4.317 +
   4.318 +source "${HOME:?}/chain.properties"
   4.319 +cat > "${HOME:?}/.metadata_env" << EOF
   4.320 +
   4.321 +POSTGRES_DB=${metadata}
   4.322 +POSTGRES_USER=${metauser}
   4.323 +POSTGRES_PASSWORD=${metapass}
   4.324 +         
   4.325 +EOF
   4.326 +
   4.327 +
   4.328 +
   4.329 +
   4.330 +## -------------------------------------------------------------------------------------------
   4.331 +## Fetch the TAP Deployer
   4.332 +## -------------------------------------------------------------------------------------------
   4.333 +
   4.334 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   4.335 +
   4.336 +
   4.337 +
   4.338 +
   4.339 +
   4.340 +
   4.341 +## -------------------------------------------------------------------------------------------
   4.342 +## Deploy Swarm stack
   4.343 +## -------------------------------------------------------------------------------------------
   4.344 +
   4.345 +docker stack deploy -c docker-compose.yml ft
   4.346 +
   4.347 +	# Creating network ft_miriam
   4.348 +	# Creating network ft_rosana
   4.349 +	# Creating service ft_jarmila
   4.350 +	# Creating service ft_gillian
   4.351 +	# Creating service ft_firethorn-py
   4.352 +	# Creating service ft_bethany
   4.353 +	# Creating service ft_carolina
   4.354 +
   4.355 +
   4.356 +
   4.357 +## -------------------------------------------------------------------------------------------
   4.358 +## To remove all docker services 
   4.359 +## -------------------------------------------------------------------------------------------
   4.360 +
   4.361 +## docker service rm $(docker service ls -q)
   4.362 +
   4.363 +
   4.364 +
   4.365 +## -------------------------------------------------------------------------------------------
   4.366 +## To list all docker services 
   4.367 +## -------------------------------------------------------------------------------------------
   4.368 +
   4.369 +## docker service ls
   4.370 +
   4.371 +
   4.372 +
   4.373 +docker ps
   4.374 +
   4.375 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                            PORTS               NAMES
   4.376 +4db9c13b1f0d        firethorn/firethorn:2.1.34      "/bin/sh -c '/var/lo…"   5 seconds ago       Up 3 seconds (health: starting)   8080/tcp            ft_gillian.1.su28yys3as7foaqdb0b9nqlwx
   4.377 +fe80420092a1        firethorn/ogsadai:2.1.34        "/bin/sh -c '/var/lo…"   8 seconds ago       Up 6 seconds (health: starting)   8080/tcp            ft_jarmila.1.0dbei14did74h1iu7ku93gx7a
   4.378 +a2c23f565cb8        firethorn/postgres:2.1.34       "docker-entrypoint.s…"   10 seconds ago      Up 8 seconds                      5432/tcp            ft_carolina.1.wx5wtw407thlr128yvh5t9lhm
   4.379 +a85761628bbb        firethorn/postgres:2.1.34       "docker-entrypoint.s…"   12 seconds ago      Up 8 seconds                      5432/tcp            ft_bethany.1.xd20qmguubfyopfy9b7sin4sk
   4.380 +5adc907ffcee        firethorn/firethorn-py:2.1.34   "python3"                15 seconds ago      Up 13 seconds                                         ft_firethorn-py.1.gc6fl7q80yr231lr0kd0pesql
   4.381 +
   4.382 +
   4.383 +
   4.384 +## ------------------------------------------------------------------------
   4.385 +## Create TAP service with firethorn-utils
   4.386 +## ------------------------------------------------------------------------
   4.387 +
   4.388 +docker exec -it ft_firethorn-py.1.gc6fl7q80yr231lr0kd0pesql bash
   4.389 +
   4.390 +
   4.391 +root@firethorn-py:/home# python3
   4.392 +
   4.393 +
   4.394 +import firethorn
   4.395 +import firethorn_utils.configurator as configur
   4.396 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   4.397 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   4.398 +configurator =  configur.Configurator(ft)
   4.399 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/osa-tap.json")
   4.400 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/vsa-tap.json")
   4.401 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/wsa-tap.json")
   4.402 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/ssa-tap.json")
   4.403 +
   4.404 +
   4.405 +
   4.406 +
   4.407 +## OSA
   4.408 +
   4.409 +Importing JDBC Schema: ATLASDR1
   4.410 +Importing JDBC Schema: ATLASDR2
   4.411 +Importing JDBC Schema: ATLASDR3
   4.412 +Importing JDBC Schema: ATLASDR4
   4.413 +Importing JDBC Schema: 2MASS
   4.414 +Importing JDBC Schema: 2XMM
   4.415 +Importing JDBC Schema: WISE
   4.416 +Importing JDBC Schema: VIKINGDR3
   4.417 +Importing JDBC Schema: VHSDR1
   4.418 +Importing JDBC Schema: MGC
   4.419 +Importing JDBC Schema: GLIMPSE
   4.420 +Importing JDBC Schema: FIRST
   4.421 +Importing JDBC Schema: DENIS
   4.422 +Importing JDBC Schema: VPHASDR3
   4.423 +Importing JDBC Schema: VVVDR4
   4.424 +ERROR:root:HTTP Error 404: 
   4.425 +Traceback (most recent call last):
   4.426 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   4.427 +    with urllib.request.urlopen( req, data) as response:
   4.428 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   4.429 +    return opener.open(url, data, timeout)
   4.430 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   4.431 +    response = meth(req, response)
   4.432 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   4.433 +    'http', request, response, code, msg, hdrs)
   4.434 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   4.435 +    return self._call_chain(*args)
   4.436 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   4.437 +    result = func(*args)
   4.438 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   4.439 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   4.440 +urllib.error.HTTPError: HTTP Error 404: 
   4.441 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   4.442 +Traceback (most recent call last):
   4.443 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   4.444 +    response_json_list = json.loads(response.text)
   4.445 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   4.446 +    return _default_decoder.decode(s)
   4.447 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   4.448 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   4.449 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   4.450 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   4.451 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   4.452 +Importing JDBC Schema: SDSSDR7
   4.453 +Importing JDBC Schema: SDSSDR9
   4.454 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   4.455 +
   4.456 +
   4.457 +
   4.458 +## VSA
   4.459 +
   4.460 +Importing JDBC Schema: VVVDR1
   4.461 +Importing JDBC Schema: VVVDR2
   4.462 +Importing JDBC Schema: VVVDR4
   4.463 +ERROR:root:HTTP Error 404: 
   4.464 +Traceback (most recent call last):
   4.465 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   4.466 +    with urllib.request.urlopen( req, data) as response:
   4.467 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   4.468 +    return opener.open(url, data, timeout)
   4.469 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   4.470 +    response = meth(req, response)
   4.471 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   4.472 +    'http', request, response, code, msg, hdrs)
   4.473 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   4.474 +    return self._call_chain(*args)
   4.475 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   4.476 +    result = func(*args)
   4.477 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   4.478 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   4.479 +urllib.error.HTTPError: HTTP Error 404: 
   4.480 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   4.481 +Traceback (most recent call last):
   4.482 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   4.483 +    response_json_list = json.loads(response.text)
   4.484 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   4.485 +    return _default_decoder.decode(s)
   4.486 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   4.487 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   4.488 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   4.489 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   4.490 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   4.491 +Importing JDBC Schema: VHSDR1
   4.492 +Importing JDBC Schema: VHSDR2
   4.493 +Importing JDBC Schema: VHSDR3
   4.494 +Importing JDBC Schema: VMCDR1
   4.495 +Importing JDBC Schema: VMCDR2
   4.496 +Importing JDBC Schema: VMCDR3
   4.497 +Importing JDBC Schema: VMCDR4
   4.498 +Importing JDBC Schema: VIKINGDR2
   4.499 +Importing JDBC Schema: VIKINGDR3
   4.500 +Importing JDBC Schema: VIKINGDR4
   4.501 +Importing JDBC Schema: VIDEODR2
   4.502 +Importing JDBC Schema: VIDEODR3
   4.503 +Importing JDBC Schema: VIDEODR4
   4.504 +Importing JDBC Schema: VIDEODR5
   4.505 +Importing JDBC Schema: UKIDSSDR4PLUS
   4.506 +Importing JDBC Schema: UKIDSSDR5PLUS
   4.507 +Importing JDBC Schema: UKIDSSDR6PLUS
   4.508 +Importing JDBC Schema: UKIDSSDR7PLUS
   4.509 +Importing JDBC Schema: UKIDSSDR8PLUS
   4.510 +Importing JDBC Schema: UKIDSSDR9PLUS
   4.511 +Importing JDBC Schema: UKIDSSDR10PLUS
   4.512 +Importing JDBC Schema: SDSSDR1
   4.513 +Importing JDBC Schema: SDSSDR3
   4.514 +Importing JDBC Schema: SDSSDR5
   4.515 +Importing JDBC Schema: SDSSDR7
   4.516 +Importing JDBC Schema: SDSSDR8
   4.517 +Importing JDBC Schema: SDSSDR9
   4.518 +Importing JDBC Schema: 2MASS
   4.519 +Importing JDBC Schema: ROSAT
   4.520 +Importing JDBC Schema: FIRST
   4.521 +Importing JDBC Schema: IRAS
   4.522 +Importing JDBC Schema: SSA
   4.523 +Importing JDBC Schema: GLIMPSE
   4.524 +Importing JDBC Schema: 2XMM
   4.525 +Importing JDBC Schema: MGC
   4.526 +Importing JDBC Schema: Stripe82
   4.527 +Importing JDBC Schema: WISE
   4.528 +TAP Service available at: http://gillian:8080/firethorn/tap/57/
   4.529 +
   4.530 +
   4.531 +
   4.532 +
   4.533 +## WSA 
   4.534 +
   4.535 +Importing JDBC Schema: UKIDSSDR1PLUS
   4.536 +Importing JDBC Schema: UKIDSSDR2PLUS
   4.537 +Importing JDBC Schema: UKIDSSDR3PLUS
   4.538 +Importing JDBC Schema: UKIDSSDR4PLUS
   4.539 +Importing JDBC Schema: UKIDSSDR5PLUS
   4.540 +Importing JDBC Schema: UKIDSSDR6PLUS
   4.541 +Importing JDBC Schema: UKIDSSDR7PLUS
   4.542 +Importing JDBC Schema: UKIDSSDR8PLUS
   4.543 +Importing JDBC Schema: UKIDSSDR9PLUS
   4.544 +Importing JDBC Schema: UKIDSSDR10PLUS
   4.545 +Importing JDBC Schema: SDSSDR1
   4.546 +Importing JDBC Schema: SDSSDR3
   4.547 +Importing JDBC Schema: SDSSDR5
   4.548 +Importing JDBC Schema: SDSSDR7
   4.549 +Importing JDBC Schema: SDSSDR8
   4.550 +Importing JDBC Schema: SDSSDR9
   4.551 +Importing JDBC Schema: 2MASS
   4.552 +Importing JDBC Schema: ROSAT
   4.553 +Importing JDBC Schema: FIRST
   4.554 +Importing JDBC Schema: IRAS
   4.555 +Importing JDBC Schema: SSA
   4.556 +Importing JDBC Schema: GLIMPSE
   4.557 +Importing JDBC Schema: 2XMM
   4.558 +Importing JDBC Schema: MGC
   4.559 +Importing JDBC Schema: Stripe82
   4.560 +Importing JDBC Schema: WISE
   4.561 +Importing JDBC Schema: WFCAMCAL08B
   4.562 +TAP Service available at: http://gillian:8080/firethorn/tap/60/
   4.563 +
   4.564 +
   4.565 +## SSA
   4.566 +
   4.567 +Importing JDBC Schema: SSA
   4.568 +Importing JDBC Schema: 2MASS
   4.569 +Importing JDBC Schema: 2MPZ
   4.570 +Importing JDBC Schema: WISExSCOSPZ
   4.571 +Importing JDBC Schema: SDSSDR1
   4.572 +Importing JDBC Schema: SDSSDR5
   4.573 +Importing JDBC Schema: SDSSDR7
   4.574 +Importing JDBC Schema: SDSSDR8
   4.575 +Importing JDBC Schema: SDSSDR9
   4.576 +Importing JDBC Schema: 6dF
   4.577 +TAP Service available at: http://gillian:8080/firethorn/tap/63/
   4.578 +
   4.579 +
   4.580 +
   4.581 +
   4.582 +
   4.583 +
   4.584 +# -----------------------------------------------------
   4.585 +# WFAU TAP Service endpoints
   4.586 +# -----------------------------------------------------
   4.587 +
   4.588 +TAP Service available at: http://gillian:8080/firethorn/tap/54/ - OSA
   4.589 +TAP Service available at: http://gillian:8080/firethorn/tap/57/ - VSA
   4.590 +TAP Service available at: http://gillian:8080/firethorn/tap/60/ - WSA
   4.591 +TAP Service available at: http://gillian:8080/firethorn/tap/63/ - SSA
   4.592 +
   4.593 +
   4.594 +
   4.595 +# -----------------------------------------------------
   4.596 +# Update the Proxy Entry
   4.597 +# -----------------------------------------------------
   4.598 +ssh Stevedore@Acilamwen 
   4.599 +
   4.600 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   4.601 +
   4.602 +..
   4.603 +    ## PROXY ENTRIES HERE ##
   4.604 +    ProxyPassMatch    ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   4.605 +    ProxyPassReverse  ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   4.606 +
   4.607 +    ProxyPassMatch    ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   4.608 +    ProxyPassReverse  ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   4.609 +
   4.610 +    ProxyPassMatch    ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   4.611 +    ProxyPassReverse  ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   4.612 +
   4.613 +    ProxyPassMatch    ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   4.614 +    ProxyPassReverse  ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   4.615 +
   4.616 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   4.617 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   4.618 +
   4.619 +
   4.620 +...   
   4.621 +
   4.622 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   4.623 +[Stevedore@Acilamwen ~]$ docker exec -it  apache bash
   4.624 +[root@bc8d6979e451 /]# nano /etc/httpd/conf.d/tap.roe.ac.uk.conf
   4.625 +
   4.626 +..
   4.627 +
   4.628 +    ## PROXY ENTRIES HERE ##
   4.629 +    ProxyPassMatch    ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   4.630 +    ProxyPassReverse  ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   4.631 +
   4.632 +    ProxyPassMatch    ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   4.633 +    ProxyPassReverse  ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   4.634 +
   4.635 +    ProxyPassMatch    ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   4.636 +    ProxyPassReverse  ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   4.637 +
   4.638 +    ProxyPassMatch    ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   4.639 +    ProxyPassReverse  ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   4.640 +
   4.641 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   4.642 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   4.643 +
   4.644 +
   4.645 +
   4.646 +..
   4.647 +[root@bc8d6979e451 /]# exit
   4.648 +exit
   4.649 +
   4.650 +[Stevedore@Acilamwen ~]$ docker kill --signal 'USR1' 'apache'
   4.651 +
     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/doc/notes/stv/20190720-Test-Firethorn-Unresponsive.txt	Wed Oct 09 16:54:28 2019 +0200
     5.3 @@ -0,0 +1,118 @@
     5.4 +#
     5.5 +# <meta:header>
     5.6 +#   <meta:licence>
     5.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     5.8 +#
     5.9 +#     This information is free software: you can redistribute it and/or modify
    5.10 +#     it under the terms of the GNU General Public License as published by
    5.11 +#     the Free Software Foundation, either version 3 of the License, or
    5.12 +#     (at your option) any later version.
    5.13 +#
    5.14 +#     This information is distributed in the hope that it will be useful,
    5.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    5.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    5.17 +#     GNU General Public License for more details.
    5.18 +#  
    5.19 +#     You should have received a copy of the GNU General Public License
    5.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    5.21 +#   </meta:licence>
    5.22 +# </meta:header>
    5.23 +#
    5.24 +#
    5.25 +
    5.26 +############################################################################
    5.27 +#####     Test to cause Firethorn Service to become unresponsive       #####
    5.28 +############################################################################
    5.29 +
    5.30 +
    5.31 +
    5.32 +## Test tap.roe.ac.uk on local machine
    5.33 +
    5.34 +## ------------------------------------------------------------------------
    5.35 +## Run Container on local machine , test Firethorn on a public IP
    5.36 +## ------------------------------------------------------------------------
    5.37 +
    5.38 +docker run -it firethorn/firethorn-py:2.1.34 python3
    5.39 +
    5.40 +
    5.41 +
    5.42 +import os                                                                       
    5.43 +from multiprocessing import Pool                                                
    5.44 +
    5.45 +process_list = range(100)
    5.46 +	
    5.47 +def run_process(process_counter):                                                             
    5.48 +    print ("Starting RUN # " + str(process_counter))
    5.49 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=54 -u= -p= -g= -v=True -m=sync')
    5.50 +    print ("Ended RUN # " + str(process_counter))
    5.51 +
    5.52 +
    5.53 +pool = Pool(processes=20)                                                        
    5.54 +pool.map(run_process, process_list)   
    5.55 +
    5.56 +
    5.57 +## Leave running for up to a few hours
    5.58 +
    5.59 +
    5.60 +
    5.61 +
    5.62 +
    5.63 +
    5.64 +
    5.65 +## Test development deployment
    5.66 +
    5.67 +## ------------------------------------------------------------------------
    5.68 +## Run Container on local machine , test Firethorn on a public IP
    5.69 +## ------------------------------------------------------------------------
    5.70 +
    5.71 +Create a Firethorn chain as described:
    5.72 +http://wfau.metagrid.co.uk/code/firethorn/file/9fd44371154b/doc/notes/stv/20190718-Firethorn-Swarm.txt
    5.73 +
    5.74 +
    5.75 +## Fetch the container name of firethorn.py
    5.76 +
    5.77 +
    5.78 +docker ps
    5.79 +CONTAINER ID        IMAGE                                     COMMAND                  CREATED             STATUS                  PORTS               NAMES
    5.80 +4eb9467315fd        firethorn/firethorn:latest                "/bin/sh -c '/var/lo…"   21 hours ago        Up 21 hours (healthy)   8080/tcp            ft_gillian.1.mqv7eoh2gdb0ydy7ektlzc6iu
    5.81 +c95dc31a37d7        firethorn/ogsadai:2.1.33-stv-tap-swarm    "/bin/sh -c '/var/lo…"   21 hours ago        Up 21 hours (healthy)   8080/tcp            ft_jarmila.1.v8rbyw4x3xkfnwpluu9kwi8nf
    5.82 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm   "docker-entrypoint.s…"   46 hours ago        Up 46 hours             5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
    5.83 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm   "docker-entrypoint.s…"   46 hours ago        Up 46 hours             5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
    5.84 +59b0ce098117        firethorn/firethorn-py:latest             "python3"                46 hours ago        Up 46 hours                                 ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
    5.85 +
    5.86 +
    5.87 +
    5.88 +
    5.89 +## Bash into Firethorn.py container
    5.90 +
    5.91 +docker exec -it ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj bash
    5.92 +
    5.93 +
    5.94 +
    5.95 +
    5.96 +root@firethorn-py:/home# python3
    5.97 +Python 3.5.2 (default, Nov 12 2018, 13:43:14) 
    5.98 +[GCC 5.4.0 20160609] on linux
    5.99 +Type "help", "copyright", "credits" or "license" for more information.
   5.100 +>>> 
   5.101 +
   5.102 +
   5.103 +
   5.104 +import os                                                                       
   5.105 +from multiprocessing import Pool                                                
   5.106 +
   5.107 +process_list = range(100)
   5.108 +	
   5.109 +def run_process(process_counter):                                                             
   5.110 +    print ("Starting RUN # " + str(process_counter))
   5.111 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://gillian:8080/firethorn -r=54 -u=chae1aixuh5exae5Udoh -p=EiB0shoegahfeWahx1ai -g=oowe1eedaeFei1tohtai -v=True -m=sync')
   5.112 +    print ("Ended RUN # " + str(process_counter))
   5.113 +
   5.114 +
   5.115 +pool = Pool(processes=20)                                                        
   5.116 +pool.map(run_process, process_list)  
   5.117 +
   5.118 +
   5.119 +
   5.120 +## Leave running until all queries start failing
   5.121 +
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/doc/notes/stv/20190828-VSA-Swarm-deploy.txt	Wed Oct 09 16:54:28 2019 +0200
     6.3 @@ -0,0 +1,664 @@
     6.4 +#
     6.5 +# <meta:header>
     6.6 +#   <meta:licence>
     6.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     6.8 +#
     6.9 +#     This information is free software: you can redistribute it and/or modify
    6.10 +#     it under the terms of the GNU General Public License as published by
    6.11 +#     the Free Software Foundation, either version 3 of the License, or
    6.12 +#     (at your option) any later version.
    6.13 +#
    6.14 +#     This information is distributed in the hope that it will be useful,
    6.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    6.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    6.17 +#     GNU General Public License for more details.
    6.18 +#  
    6.19 +#     You should have received a copy of the GNU General Public License
    6.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    6.21 +#   </meta:licence>
    6.22 +# </meta:header>
    6.23 +#
    6.24 +#
    6.25 +
    6.26 +
    6.27 +## ------------------------------------------------------------------------------------------
    6.28 +## Create Master VM 
    6.29 +## ------------------------------------------------------------------------------------------
    6.30 +
    6.31 +stv@trop02:~$ createvm 
    6.32 +
    6.33 +INFO : Base pool  [base]
    6.34 +INFO : Live pool  [live]
    6.35 +INFO : Connection [qemu:///system]
    6.36 +
    6.37 +INFO : Data path  [/var/local/projects/ischnura/github/src/dat]
    6.38 +
    6.39 +INFO : Machines   [/var/local/projects/ischnura/github/src/dat/tropo-machines.txt]
    6.40 +INFO : Template   [/var/local/projects/ischnura/github/src/dat/tropo-template.xml]
    6.41 +
    6.42 +[1] Araybwyn
    6.43 +[2] Lothigometh
    6.44 +[3] Ulov
    6.45 +[4] Dwiema
    6.46 +[5] Ibalehar
    6.47 +[6] Eterathiel
    6.48 +[7] Siamond
    6.49 +[8] Acilamwen
    6.50 +Select machine name (1) 3
    6.51 +
    6.52 +[1] fedora-28-docker-base-20180708.qcow
    6.53 +[2] fedora-28-8G-docker-base-20181016.qcow
    6.54 +[3] fedora-28-32G-docker-base-20181016.qcow
    6.55 +[4] fedora-28-16G-docker-base-20181016.qcow
    6.56 +Select base image (1) 3
    6.57 +
    6.58 +INFO : Node name [Ulov]
    6.59 +INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    6.60 +INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    6.61 +INFO : Disc name [Ulov.qcow]
    6.62 +INFO : Disc size [32GiB]
    6.63 +
    6.64 +INFO : MAC  [52:54:00:02:02:0A]
    6.65 +INFO : IPv4 [192.168.202.10]
    6.66 +INFO : IPv6 []
    6.67 +
    6.68 +Create virtual machine (Y/n)Y
    6.69 +
    6.70 +Found existing storage volume [Ulov.qcow]
    6.71 +Delete storage volume (Y/n)Y
    6.72 +Deleting storage volume [Ulov.qcow]
    6.73 +Vol Ulov.qcow deleted
    6.74 +
    6.75 +Creating new volume [Ulov.qcow]
    6.76 +Vol Ulov.qcow created
    6.77 +Name:           Ulov.qcow
    6.78 +Type:           file
    6.79 +Capacity:       32.00 GiB
    6.80 +Allocation:     196.00 KiB
    6.81 +
    6.82 +Vol Ulov.iso deleted
    6.83 +Vol Ulov.iso created
    6.84 +Creating new virtual machine [Ulov]
    6.85 +Domain Ulov defined from /tmp/tmp.3mwcAcOuPd
    6.86 +Starting new virtual machine [Ulov]
    6.87 +Domain Ulov started
    6.88 +
    6.89 +
    6.90 +
    6.91 +
    6.92 +## ------------------------------------------------------------------------------------------
    6.93 +## Initialize Swarm with VM's address
    6.94 +## ------------------------------------------------------------------------------------------
    6.95 +
    6.96 +[Stevedore@Ulov ~]$ ifconfig
    6.97 +
    6.98 +       ..  192.168.202.10  ..
    6.99 +
   6.100 +
   6.101 +docker swarm init --advertise-addr 192.168.202.10  
   6.102 +
   6.103 +
   6.104 +	To add a worker to this swarm, run the following command:
   6.105 +
   6.106 +	    docker swarm join --token SWMTKN-1-3qmr0ckemiw0rb4wnuss9of7j7xxhjs4nze7nauoiyh9oe9pfv-cjvopumypyb62jqonb1nbx5a0 192.168.202.10:2377
   6.107 +
   6.108 +	To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
   6.109 +
   6.110 +
   6.111 +
   6.112 +## Fix Firewall issue
   6.113 +sudo su
   6.114 +    firewall-cmd --add-port=2377/tcp --permanent
   6.115 +    firewall-cmd --add-port=2377/tcp --permanent
   6.116 +    firewall-cmd --add-port=7946/tcp --permanent
   6.117 +    firewall-cmd --add-port=4789/tcp --permanent
   6.118 +    firewall-cmd --add-port=4789/udp --permanent
   6.119 +    iptables -A INPUT -p 50 -j ACCEPT    
   6.120 +    firewall-cmd --reload
   6.121 +exit
   6.122 +
   6.123 +
   6.124 +## -------------------------------------------------------------------------------------------
   6.125 +## Add secret function.
   6.126 +## -------------------------------------------------------------------------------------------
   6.127 +
   6.128 +    secrethost='stv@shepseskaf.roe.ac.uk'
   6.129 +    secretfile='${HOME:?}/secret.store.20190828'
   6.130 +
   6.131 +    secret()
   6.132 +    {
   6.133 +        local key=${1:?}
   6.134 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
   6.135 +    }
   6.136 +
   6.137 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
   6.138 +
   6.139 +
   6.140 +
   6.141 +## -------------------------------------------------------------------------------------------
   6.142 +## Set the target branch and builtag
   6.143 +## -------------------------------------------------------------------------------------------
   6.144 +
   6.145 +
   6.146 +    export buildtag=2.1.34
   6.147 +    export branch=default
   6.148 +
   6.149 +
   6.150 +
   6.151 +## -------------------------------------------------------------------------------------------
   6.152 +## Create our chain.properties file
   6.153 +## -------------------------------------------------------------------------------------------
   6.154 +
   6.155 +cat > "${HOME:?}/chain.properties" << EOF
   6.156 +
   6.157 +metadata=postgres
   6.158 +metauser=$(pwgen 20 1)
   6.159 +metapass=$(pwgen 20 1) 
   6.160 +
   6.161 +userhost=$(secret 'firethorn.user.host')
   6.162 +userdata=$(secret 'firethorn.user.data')
   6.163 +useruser=$(secret 'firethorn.user.user')
   6.164 +userpass=$(secret 'firethorn.user.pass')
   6.165 +usertype=mssql
   6.166 +
   6.167 +datahost=$(secret 'firethorn.data.host')
   6.168 +datadata=$(secret 'firethorn.data.data')
   6.169 +datacatalog=$(secret 'firethorn.data.catalog')
   6.170 +datatype=$(secret 'firethorn.data.type')
   6.171 +datauser=$(secret 'firethorn.data.user')
   6.172 +datapass=$(secret 'firethorn.data.pass')
   6.173 +
   6.174 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   6.175 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   6.176 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   6.177 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   6.178 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   6.179 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   6.180 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   6.181 +
   6.182 +tunneluser=$(secret 'ssh.tunnel.user')
   6.183 +tunnelhost=$(secret 'ssh.tunnel.host')
   6.184 +
   6.185 +admingroup=$(pwgen 20 1)	
   6.186 +adminuser=$(pwgen 20 1)
   6.187 +adminpass=$(pwgen 20 1)
   6.188 +
   6.189 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   6.190 + 
   6.191 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   6.192 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   6.193 +
   6.194 +guestgroup=friends
   6.195 +endpoint=$(secret 'endpoint')
   6.196 +
   6.197 +firethornhost=gillian
   6.198 +
   6.199 +EOF
   6.200 +
   6.201 +
   6.202 +
   6.203 +## -----------------------------------------------------
   6.204 +## Create our Firethorn properties.
   6.205 +## -----------------------------------------------------
   6.206 +
   6.207 +source "${HOME:?}/chain.properties"
   6.208 +cat > "${HOME:?}/firethorn.properties" << EOF
   6.209 +
   6.210 +firethorn.meta.type=pgsql
   6.211 +firethorn.meta.driver=org.postgresql.Driver
   6.212 +firethorn.meta.host=bethany
   6.213 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   6.214 +firethorn.meta.data=postgres
   6.215 +firethorn.meta.user=${metauser:?}
   6.216 +firethorn.meta.pass=${metapass:?}
   6.217 +
   6.218 +firethorn.user.type=mssql
   6.219 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   6.220 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   6.221 +firethorn.user.host=${userhost:?}
   6.222 +firethorn.user.data=${userdata:?}
   6.223 +firethorn.user.user=${useruser:?}
   6.224 +firethorn.user.pass=${userpass:?}
   6.225 +
   6.226 +firethorn.tapschema.resource.name=${tapschemajdbc}
   6.227 +firethorn.tapschema.database.name=${tapschemadata}
   6.228 +firethorn.tapschema.database.host=${tapschemahost}
   6.229 +firethorn.tapschema.database.port=${tapschemaport}
   6.230 +firethorn.tapschema.database.user=${tapschemauser}
   6.231 +firethorn.tapschema.database.pass=${tapschemapass}
   6.232 +
   6.233 +firethorn.admin.community=${admingroup:?}
   6.234 +firethorn.admin.user.name=${adminuser:?}
   6.235 +firethorn.admin.user.pass=${adminpass:?}
   6.236 +
   6.237 +firethorn.guest.community=${guestgroup:?}
   6.238 +
   6.239 +adql.query.delay.first=${adql_query_delay_first:?}
   6.240 +
   6.241 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   6.242 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   6.243 +
   6.244 +firethornhost=gillian
   6.245 +
   6.246 +
   6.247 +EOF
   6.248 +
   6.249 +
   6.250 +
   6.251 +## -------------------------------------------------------------------------------------------
   6.252 +## Create our compose env files.
   6.253 +## -------------------------------------------------------------------------------------------
   6.254 +
   6.255 +source "${HOME:?}/chain.properties"
   6.256 +cat > "${HOME:?}/.env" << EOF
   6.257 +
   6.258 +buildtag=${branch:?}
   6.259 +endpoint=${endpoint:?}
   6.260 +
   6.261 +
   6.262 +metadata=postgres
   6.263 +metauser=${metauser:?}
   6.264 +metapass=${metapass:?}
   6.265 +firethorn.meta.type=pgsql
   6.266 +firethorn.meta.driver=org.postgresql.Driver
   6.267 +firethorn.meta.host=bethany
   6.268 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   6.269 +firethorn.meta.data=postgres
   6.270 +firethorn.meta.user=${metauser:?}
   6.271 +firethorn.meta.pass=${metapass:?}
   6.272 +
   6.273 +
   6.274 +datadata=${datadata:?}
   6.275 +datauser=${datauser:?}
   6.276 +datapass=${datapass:?}
   6.277 +datahost=${datahost:?}
   6.278 +datacatalog=${datacatalog:?}
   6.279 +datatype=mssql
   6.280 +
   6.281 +
   6.282 +tapschemadata=${tapschemadata:?}
   6.283 +tapschemauser=${tapschemauser:?}
   6.284 +tapschemapass=${tapschemapass:?}
   6.285 +tapschemahost=${tapschemahost}
   6.286 +tapschemaport=${tapschemaport:?}
   6.287 +tapschemajdbc=${tapschemajdbc:?}
   6.288 +tapschematype=${tapschematype:?}
   6.289 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   6.290 +firethorn.tapschema.database.name=${tapschemadata:?}
   6.291 +firethorn.tapschema.database.host=${tapschemahost:?}
   6.292 +firethorn.tapschema.database.port=5432
   6.293 +firethorn.tapschema.database.user=${tapschemauser:?}
   6.294 +firethorn.tapschema.database.pass=${tapschemapass:?}
   6.295 +firethorn.tapschema.database.type=pgsql
   6.296 +
   6.297 +userhost=${userhost:?}
   6.298 +userdata=${userdata:?}
   6.299 +useruser=${useruser:?}
   6.300 +userpass=${userpass:?}
   6.301 +usertype=mssql
   6.302 +firethorn.user.type=mssql
   6.303 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   6.304 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   6.305 +firethorn.user.host=${userhost:?}
   6.306 +firethorn.user.data=${userdata:?}
   6.307 +firethorn.user.user=${useruser:?}
   6.308 +firethorn.user.pass=${userpass:?}
   6.309 +
   6.310 +
   6.311 +admingroup=${admingroup:?}
   6.312 +adminuser=${adminuser:?}
   6.313 +adminpass=${adminpass:?}
   6.314 +guestgroup=${guestgroup:?}
   6.315 +firethorn.admin.community=${admingroup:?}
   6.316 +firethorn.admin.user.name=${adminuser:?}
   6.317 +firethorn.admin.user.pass=${adminpass:?}
   6.318 +firethorn.guest.community=${guestgroup:?}
   6.319 +
   6.320 +
   6.321 +adql_query_delay_first=${adql_query_delay_first:?}
   6.322 +adql.query.delay.first=${adql_query_delay_first:?}
   6.323 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   6.324 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   6.325 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   6.326 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   6.327 +
   6.328 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   6.329 +firethornhost=gillian
   6.330 +
   6.331 +
   6.332 +EOF
   6.333 +
   6.334 +
   6.335 +source "${HOME:?}/chain.properties"
   6.336 +cat > "${HOME:?}/.tapschema_env" << EOF
   6.337 +
   6.338 +POSTGRES_DB=${tapschemadata}
   6.339 +POSTGRES_USER=${tapschemauser}
   6.340 +POSTGRES_PASSWORD=${tapschemapass}
   6.341 +
   6.342 +EOF
   6.343 +
   6.344 +
   6.345 +source "${HOME:?}/chain.properties"
   6.346 +cat > "${HOME:?}/.metadata_env" << EOF
   6.347 +
   6.348 +POSTGRES_DB=${metadata}
   6.349 +POSTGRES_USER=${metauser}
   6.350 +POSTGRES_PASSWORD=${metapass}
   6.351 +         
   6.352 +EOF
   6.353 +
   6.354 +
   6.355 +## -------------------------------------------------------------------------------------------
   6.356 +## Fetch the TAP Deployer
   6.357 +## -------------------------------------------------------------------------------------------
   6.358 +
   6.359 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   6.360 +
   6.361 +
   6.362 +
   6.363 +## ------------------------------------------------------------------------
   6.364 +## Deploy Swarm stack
   6.365 +## ------------------------------------------------------------------------
   6.366 +
   6.367 +docker stack deploy -c docker-compose.yml ft
   6.368 +
   6.369 +
   6.370 +
   6.371 +## ------------------------------------------------------------------------
   6.372 +## To remove all docker services 
   6.373 +## ------------------------------------------------------------------------
   6.374 +
   6.375 +## docker service rm $(docker service ls -q)
   6.376 +
   6.377 +
   6.378 +
   6.379 +## ------------------------------------------------------------------------
   6.380 +## To remove stack of services
   6.381 +## ------------------------------------------------------------------------
   6.382 +
   6.383 +## docker stack rm ft
   6.384 +
   6.385 +
   6.386 +[Stevedore@Lothigometh ~]$ docker ps
   6.387 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                   PORTS               NAMES
   6.388 +a6b8ce6fc1a7        firethorn/firethorn-py:2.1.35   "python3"                2 minutes ago       Up 2 minutes                                 ft_firethorn-py.1.b6ww5ws0x5hsoxzibvvmzkpg2
   6.389 +e2aa39f93f71        firethorn/ogsadai:2.1.35        "/bin/sh -c '/var/lo…"   3 minutes ago       Up 2 minutes (healthy)   8080/tcp            ft_jarmila.1.k0lafcd3mnk4omqqa6b3sfgga
   6.390 +77839250c9d9        firethorn/firethorn:2.1.35      "/bin/sh -c '/var/lo…"   3 minutes ago       Up 2 minutes (healthy)   8080/tcp            ft_gillian.1.b375fvmymiqzubvwrqi3jcp0u
   6.391 +754c7061e9a7        firethorn/postgres:2.1.35       "docker-entrypoint.s…"   3 minutes ago       Up 3 minutes             5432/tcp            ft_carolina.1.ry40mzo4vy2kipfj8s879ak95
   6.392 +166cb66e3bb1        firethorn/postgres:2.1.35       "docker-entrypoint.s…"   3 minutes ago       Up 3 minutes             5432/tcp            ft_bethany.1.jlh7gdn8oau9dcxhhmd9kafkn
   6.393 +
   6.394 +
   6.395 +
   6.396 +
   6.397 +## ------------------------------------------------------------------------
   6.398 +## Run Docker Compose tap-deployer.yml from firethorn.py
   6.399 +## ------------------------------------------------------------------------
   6.400 +
   6.401 +docker exec -it fa6b8ce6fc1a7 bash
   6.402 +
   6.403 +python3
   6.404 +
   6.405 +import firethorn
   6.406 +import firethorn_utils.configurator as configur
   6.407 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   6.408 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   6.409 +configurator =  configur.Configurator(ft)
   6.410 +
   6.411 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   6.412 +Importing JDBC Schema: ATLASDR1
   6.413 +Importing JDBC Schema: ATLASDR2
   6.414 +Importing JDBC Schema: ATLASDR3
   6.415 +Importing JDBC Schema: 2MASS
   6.416 +Importing JDBC Schema: 2XMM
   6.417 +Importing JDBC Schema: WISE
   6.418 +Importing JDBC Schema: VIKINGDR3
   6.419 +Importing JDBC Schema: VHSDR1
   6.420 +Importing JDBC Schema: MGC
   6.421 +Importing JDBC Schema: GLIMPSE
   6.422 +Importing JDBC Schema: FIRST
   6.423 +Importing JDBC Schema: DENIS
   6.424 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   6.425 +
   6.426 +
   6.427 +configurator.load_resources("/home/firethorn.py/firethorn/data/ssa-tap.json")
   6.428 +>>> configurator.load_resources("/home/firethorn.py/firethorn/data/ssa-tap.json")
   6.429 +Importing JDBC Schema: SSA
   6.430 +Importing JDBC Schema: 2MASS
   6.431 +Importing JDBC Schema: 2MPZ
   6.432 +Importing JDBC Schema: WISExSCOSPZ
   6.433 +Importing JDBC Schema: SDSSDR1
   6.434 +Importing JDBC Schema: SDSSDR5
   6.435 +Importing JDBC Schema: SDSSDR7
   6.436 +Importing JDBC Schema: SDSSDR8
   6.437 +Importing JDBC Schema: SDSSDR9
   6.438 +Importing JDBC Schema: 6dF
   6.439 +TAP Service available at: http://gillian:8080/firethorn/tap/57/
   6.440 +
   6.441 +configurator.load_resources("/home/firethorn.py/firethorn/data/vsa-tap.json")
   6.442 +>>> configurator.load_resources("/home/firethorn.py/firethorn/data/vsa-tap.json")
   6.443 +Importing JDBC Schema: VVVDR1
   6.444 +Importing JDBC Schema: VVVDR2
   6.445 +Importing JDBC Schema: VVVDR4
   6.446 +Importing JDBC Schema: VVVDR5
   6.447 +Importing JDBC Schema: VHSDR1
   6.448 +Importing JDBC Schema: VHSDR2
   6.449 +Importing JDBC Schema: VHSDR3
   6.450 +Importing JDBC Schema: VHSDR6
   6.451 +Importing JDBC Schema: VMCDR1
   6.452 +Importing JDBC Schema: VMCDR2
   6.453 +Importing JDBC Schema: VMCDR3
   6.454 +Importing JDBC Schema: VMCDR4
   6.455 +Importing JDBC Schema: VIKINGDR2
   6.456 +Importing JDBC Schema: VIKINGDR3
   6.457 +Importing JDBC Schema: VIKINGDR4
   6.458 +Importing JDBC Schema: VIDEODR2
   6.459 +Importing JDBC Schema: VIDEODR3
   6.460 +Importing JDBC Schema: VIDEODR4
   6.461 +Importing JDBC Schema: VIDEODR5
   6.462 +Importing JDBC Schema: UKIDSSDR4PLUS
   6.463 +Importing JDBC Schema: UKIDSSDR5PLUS
   6.464 +Importing JDBC Schema: UKIDSSDR6PLUS
   6.465 +Importing JDBC Schema: UKIDSSDR7PLUS
   6.466 +Importing JDBC Schema: UKIDSSDR8PLUS
   6.467 +Importing JDBC Schema: UKIDSSDR9PLUS
   6.468 +Importing JDBC Schema: UKIDSSDR10PLUS
   6.469 +Importing JDBC Schema: SDSSDR1
   6.470 +Importing JDBC Schema: SDSSDR3
   6.471 +Importing JDBC Schema: SDSSDR5
   6.472 +Importing JDBC Schema: SDSSDR7
   6.473 +Importing JDBC Schema: SDSSDR8
   6.474 +Importing JDBC Schema: SDSSDR9
   6.475 +Importing JDBC Schema: 2MASS
   6.476 +Importing JDBC Schema: ROSAT
   6.477 +Importing JDBC Schema: FIRST
   6.478 +Importing JDBC Schema: IRAS
   6.479 +Importing JDBC Schema: SSA
   6.480 +Importing JDBC Schema: GLIMPSE
   6.481 +Importing JDBC Schema: 2XMM
   6.482 +Importing JDBC Schema: MGC
   6.483 +Importing JDBC Schema: Stripe82
   6.484 +Importing JDBC Schema: WISE
   6.485 +TAP Service available at: http://gillian:8080/firethorn/tap/60/
   6.486 +
   6.487 +
   6.488 +
   6.489 +configurator.load_resources("/home/firethorn.py/firethorn/data/wsa-tap.json")
   6.490 +>>> configurator.load_resources("/home/firethorn.py/firethorn/data/wsa-tap.json")
   6.491 +Importing JDBC Schema: UKIDSSDR1PLUS
   6.492 +Importing JDBC Schema: UKIDSSDR2PLUS
   6.493 +Importing JDBC Schema: UKIDSSDR3PLUS
   6.494 +Importing JDBC Schema: UKIDSSDR4PLUS
   6.495 +Importing JDBC Schema: UKIDSSDR5PLUS
   6.496 +Importing JDBC Schema: UKIDSSDR6PLUS
   6.497 +Importing JDBC Schema: UKIDSSDR7PLUS
   6.498 +Importing JDBC Schema: UKIDSSDR8PLUS
   6.499 +Importing JDBC Schema: UKIDSSDR9PLUS
   6.500 +Importing JDBC Schema: UKIDSSDR10PLUS
   6.501 +ERROR:root:HTTP Error 404: 
   6.502 +Traceback (most recent call last):
   6.503 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   6.504 +    with urllib.request.urlopen( req, data) as response:
   6.505 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   6.506 +    return opener.open(url, data, timeout)
   6.507 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   6.508 +    response = meth(req, response)
   6.509 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   6.510 +    'http', request, response, code, msg, hdrs)
   6.511 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   6.512 +    return self._call_chain(*args)
   6.513 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   6.514 +    result = func(*args)
   6.515 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   6.516 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   6.517 +urllib.error.HTTPError: HTTP Error 404: 
   6.518 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   6.519 +Traceback (most recent call last):
   6.520 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   6.521 +    response_json_list = json.loads(response.text)
   6.522 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   6.523 +    return _default_decoder.decode(s)
   6.524 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   6.525 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   6.526 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   6.527 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   6.528 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   6.529 +Importing JDBC Schema: SDSSDR1
   6.530 +Importing JDBC Schema: SDSSDR3
   6.531 +Importing JDBC Schema: SDSSDR5
   6.532 +Importing JDBC Schema: SDSSDR7
   6.533 +Importing JDBC Schema: SDSSDR8
   6.534 +Importing JDBC Schema: SDSSDR9
   6.535 +ERROR:root:HTTP Error 404: 
   6.536 +Traceback (most recent call last):
   6.537 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   6.538 +    with urllib.request.urlopen( req, data) as response:
   6.539 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   6.540 +    return opener.open(url, data, timeout)
   6.541 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   6.542 +    response = meth(req, response)
   6.543 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   6.544 +    'http', request, response, code, msg, hdrs)
   6.545 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   6.546 +    return self._call_chain(*args)
   6.547 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   6.548 +    result = func(*args)
   6.549 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   6.550 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   6.551 +urllib.error.HTTPError: HTTP Error 404: 
   6.552 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   6.553 +Traceback (most recent call last):
   6.554 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   6.555 +    response_json_list = json.loads(response.text)
   6.556 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   6.557 +    return _default_decoder.decode(s)
   6.558 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   6.559 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   6.560 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   6.561 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   6.562 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   6.563 +Importing JDBC Schema: 2MASS
   6.564 +Importing JDBC Schema: ROSAT
   6.565 +Importing JDBC Schema: FIRST
   6.566 +Importing JDBC Schema: IRAS
   6.567 +Importing JDBC Schema: SSA
   6.568 +ERROR:root:HTTP Error 404: 
   6.569 +Traceback (most recent call last):
   6.570 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   6.571 +    with urllib.request.urlopen( req, data) as response:
   6.572 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   6.573 +    return opener.open(url, data, timeout)
   6.574 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   6.575 +    response = meth(req, response)
   6.576 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   6.577 +    'http', request, response, code, msg, hdrs)
   6.578 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   6.579 +    return self._call_chain(*args)
   6.580 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   6.581 +    result = func(*args)
   6.582 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   6.583 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   6.584 +urllib.error.HTTPError: HTTP Error 404: 
   6.585 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   6.586 +Traceback (most recent call last):
   6.587 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   6.588 +    response_json_list = json.loads(response.text)
   6.589 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   6.590 +    return _default_decoder.decode(s)
   6.591 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   6.592 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   6.593 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   6.594 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   6.595 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   6.596 +Importing JDBC Schema: GLIMPSE
   6.597 +Importing JDBC Schema: 2XMM
   6.598 +Importing JDBC Schema: MGC
   6.599 +Importing JDBC Schema: Stripe82
   6.600 +Importing JDBC Schema: WISE
   6.601 +TAP Service available at: http://gillian:8080/firethorn/tap/63/
   6.602 +
   6.603 +..
   6.604 +
   6.605 +
   6.606 +
   6.607 +
   6.608 +# -----------------------------------------------------
   6.609 +# Update the Proxy Entry
   6.610 +# -----------------------------------------------------
   6.611 +ssh Stevedore@Acilamwen 
   6.612 +
   6.613 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   6.614 +
   6.615 +..
   6.616 +
   6.617 +    ## PROXY ENTRIES HERE ##
   6.618 +    ProxyPassMatch    ^/osa\/(.*)$  http://ulov:8080/firethorn/tap/54/$1
   6.619 +    ProxyPassReverse  ^/osa\/(.*)$  http://ulov:8080/firethorn/tap/54/$1
   6.620 +
   6.621 +    ProxyPassMatch    ^/vsa\/(.*)$  http://ulov:8080/firethorn/tap/57/$1
   6.622 +    ProxyPassReverse  ^/vsa\/(.*)$  http://ulov:8080/firethorn/tap/57/$1
   6.623 +
   6.624 +    ProxyPassMatch    ^/wsa\/(.*)$  http://ulov:8080/firethorn/tap/60/$1
   6.625 +    ProxyPassReverse  ^/wsa\/(.*)$  http://ulov:8080/firethorn/tap/60/$1
   6.626 +
   6.627 +    ProxyPassMatch    ^/ssa\/(.*)$  http://ulov:8080/firethorn/tap/63/$1
   6.628 +    ProxyPassReverse  ^/ssa\/(.*)$  http://ulov:8080/firethorn/tap/63/$1
   6.629 +
   6.630 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://ulov:8080/firethorn/$1
   6.631 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://ulov:8080/firethorn/$1
   6.632 +
   6.633 +
   6.634 +
   6.635 +...   
   6.636 +
   6.637 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   6.638 +[Stevedore@Acilamwen ~]$ docker exec -it  apache bash
   6.639 +[root@bc8d6979e451 /]# nano /etc/httpd/conf.d/tap.roe.ac.uk.conf
   6.640 +
   6.641 +..
   6.642 +
   6.643 +
   6.644 +    ## PROXY ENTRIES HERE ##
   6.645 +    ProxyPassMatch    ^/osa\/(.*)$  http://ulov:8080/firethorn/tap/54/$1
   6.646 +    ProxyPassReverse  ^/osa\/(.*)$  http://ulov:8080/firethorn/tap/54/$1
   6.647 +
   6.648 +    ProxyPassMatch    ^/vsa\/(.*)$  http://ulov:8080/firethorn/tap/57/$1
   6.649 +    ProxyPassReverse  ^/vsa\/(.*)$  http://ulov:8080/firethorn/tap/57/$1
   6.650 +
   6.651 +    ProxyPassMatch    ^/wsa\/(.*)$  http://ulov:8080/firethorn/tap/60/$1
   6.652 +    ProxyPassReverse  ^/wsa\/(.*)$  http://ulov:8080/firethorn/tap/60/$1
   6.653 +
   6.654 +    ProxyPassMatch    ^/ssa\/(.*)$  http://ulov:8080/firethorn/tap/63/$1
   6.655 +    ProxyPassReverse  ^/ssa\/(.*)$  http://ulov:8080/firethorn/tap/63/$1
   6.656 +
   6.657 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://ulov:8080/firethorn/$1
   6.658 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://ulov:8080/firethorn/$1
   6.659 +
   6.660 +
   6.661 +
   6.662 +..
   6.663 +[root@bc8d6979e451 /]# exit
   6.664 +exit
   6.665 +
   6.666 +[Stevedore@Acilamwen ~]$ docker kill --signal 'USR1' 'apache'
   6.667 +
     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/doc/notes/stv/20190903-TAP-Swarm-deploy.txt	Wed Oct 09 16:54:28 2019 +0200
     7.3 @@ -0,0 +1,702 @@
     7.4 +#
     7.5 +# <meta:header>
     7.6 +#   <meta:licence>
     7.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     7.8 +#
     7.9 +#     This information is free software: you can redistribute it and/or modify
    7.10 +#     it under the terms of the GNU General Public License as published by
    7.11 +#     the Free Software Foundation, either version 3 of the License, or
    7.12 +#     (at your option) any later version.
    7.13 +#
    7.14 +#     This information is distributed in the hope that it will be useful,
    7.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    7.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    7.17 +#     GNU General Public License for more details.
    7.18 +#  
    7.19 +#     You should have received a copy of the GNU General Public License
    7.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    7.21 +#   </meta:licence>
    7.22 +# </meta:header>
    7.23 +#
    7.24 +#
    7.25 +
    7.26 +
    7.27 +stv@trop02:~$ createvm
    7.28 +
    7.29 +INFO : Base pool  [base]
    7.30 +INFO : Live pool  [live]
    7.31 +INFO : Connection [qemu:///system]
    7.32 +
    7.33 +INFO : Data path  [/var/local/projects/ischnura/github/src/dat]
    7.34 +
    7.35 +INFO : Machines   [/var/local/projects/ischnura/github/src/dat/tropo-machines.txt]
    7.36 +INFO : Template   [/var/local/projects/ischnura/github/src/dat/tropo-template.xml]
    7.37 +
    7.38 +[1] Araybwyn
    7.39 +[2] Lothigometh
    7.40 +[3] Ulov
    7.41 +[4] Dwiema
    7.42 +[5] Ibalehar
    7.43 +[6] Eterathiel
    7.44 +[7] Siamond
    7.45 +[8] Acilamwen
    7.46 +Select machine name (1) 2
    7.47 +
    7.48 +[1] fedora-28-docker-base-20180708.qcow
    7.49 +[2] fedora-28-8G-docker-base-20181016.qcow
    7.50 +[3] fedora-28-32G-docker-base-20181016.qcow
    7.51 +[4] fedora-28-16G-docker-base-20181016.qcow
    7.52 +Select base image (1) 3
    7.53 +
    7.54 +INFO : Node name [Lothigometh]
    7.55 +INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    7.56 +INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    7.57 +INFO : Disc name [Lothigometh.qcow]
    7.58 +INFO : Disc size [32GiB]
    7.59 +
    7.60 +INFO : MAC  [52:54:00:02:02:09]
    7.61 +INFO : IPv4 [192.168.202.9]
    7.62 +INFO : IPv6 []
    7.63 +
    7.64 +Create virtual machine (Y/n)Y
    7.65 +
    7.66 +Found existing virtual machine [Lothigometh]
    7.67 +Delete existing virtual machine (Y/n)Y
    7.68 +Stopping virtual machine [Lothigometh]
    7.69 +Domain a22a63dd-3a36-471a-b24a-4a3756abdc96 destroyed
    7.70 +Delete associated storage (Y/n)Y
    7.71 +Deleting virtual machine [Lothigometh]
    7.72 +Domain a22a63dd-3a36-471a-b24a-4a3756abdc96 has been undefined
    7.73 +Y
    7.74 +Volume 'vda'(/libvirt/storage/live/Lothigometh.qcow) removed.
    7.75 +Volume 'vdb'(/var/lib/libvirt/images/init/Lothigometh.iso) removed.
    7.76 +
    7.77 +Creating new volume [Lothigometh.qcow]
    7.78 +Vol Lothigometh.qcow created
    7.79 +Name:           Lothigometh.qcow
    7.80 +Type:           file
    7.81 +Capacity:       32.00 GiB
    7.82 +Allocation:     196.00 KiB
    7.83 +
    7.84 +Vol Lothigometh.iso created
    7.85 +Creating new virtual machine [Lothigometh]
    7.86 +Domain Lothigometh defined from /tmp/tmp.1QAPX8kigk
    7.87 +Starting new virtual machine [Lothigometh]
    7.88 +Domain Lothigometh started
    7.89 +stv@trop02:~$ Y
    7.90 +-bash: Y: command not found
    7.91 +stv@trop02:~$ exit
    7.92 +logout
    7.93 +Connection to trop02.roe.ac.uk closed.
    7.94 +
    7.95 +
    7.96 +ssh-keygen -f "/home/stelios/.ssh/known_hosts" -R "lothigometh"
    7.97 +
    7.98 +ssh Stevedore@Lothigometh
    7.99 +[Stevedore@Lothigometh ~]$ 
   7.100 +
   7.101 +
   7.102 +
   7.103 +## ------------------------------------------------------------------------------------------
   7.104 +## Initialize Swarm with VM's address
   7.105 +## ------------------------------------------------------------------------------------------
   7.106 +
   7.107 +[Stevedore@Lothigometh ~]$ ifconfig
   7.108 +
   7.109 +       ..  192.168.202.9  ..
   7.110 +
   7.111 +
   7.112 +docker swarm init --advertise-addr 192.168.202.9 
   7.113 +
   7.114 +
   7.115 +	To add a worker to this swarm, run the following command:
   7.116 +
   7.117 +	    docker swarm join --token SWMTKN-1-3qmr0ckemiw0rb4wnuss9of7j7xxhjs4nze7nauoiyh9oe9pfv-cjvopumypyb62jqonb1nbx5a0 192.168.202.10:2377
   7.118 +
   7.119 +	To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
   7.120 +
   7.121 +
   7.122 +## Firewall Ports for Swarm
   7.123 +sudo su
   7.124 +    firewall-cmd --add-port=2377/tcp --permanent
   7.125 +    firewall-cmd --add-port=2377/tcp --permanent
   7.126 +    firewall-cmd --add-port=7946/tcp --permanent
   7.127 +    firewall-cmd --add-port=4789/tcp --permanent
   7.128 +    firewall-cmd --add-port=4789/udp --permanent
   7.129 +    iptables -A INPUT -p 50 -j ACCEPT    
   7.130 +    firewall-cmd --reload
   7.131 +exit
   7.132 +
   7.133 +
   7.134 +## -------------------------------------------------------------------------------------------
   7.135 +## Add secret function.
   7.136 +## -------------------------------------------------------------------------------------------
   7.137 +
   7.138 +    secrethost='stv@shepseskaf.roe.ac.uk'
   7.139 +    secretfile='${HOME:?}/secret.store.20190828'
   7.140 +
   7.141 +    secret()
   7.142 +    {
   7.143 +        local key=${1:?}
   7.144 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
   7.145 +    }
   7.146 +
   7.147 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
   7.148 +
   7.149 +
   7.150 +
   7.151 +## -------------------------------------------------------------------------------------------
   7.152 +## Set the target branch and builtag
   7.153 +## -------------------------------------------------------------------------------------------
   7.154 +
   7.155 +
   7.156 +    export buildtag=2.1.35
   7.157 +    export branch=default
   7.158 +
   7.159 +
   7.160 +
   7.161 +## -------------------------------------------------------------------------------------------
   7.162 +## Create our chain.properties file
   7.163 +## -------------------------------------------------------------------------------------------
   7.164 +
   7.165 +cat > "${HOME:?}/chain.properties" << EOF
   7.166 +
   7.167 +metadata=postgres
   7.168 +metauser=$(pwgen 20 1)
   7.169 +metapass=$(pwgen 20 1) 
   7.170 +
   7.171 +userhost=$(secret 'firethorn.user.host')
   7.172 +userdata=$(secret 'firethorn.user.data')
   7.173 +useruser=$(secret 'firethorn.user.user')
   7.174 +userpass=$(secret 'firethorn.user.pass')
   7.175 +usertype=mssql
   7.176 +
   7.177 +datahost=$(secret 'firethorn.data.host')
   7.178 +datadata=$(secret 'firethorn.data.data')
   7.179 +datacatalog=$(secret 'firethorn.data.catalog')
   7.180 +datatype=$(secret 'firethorn.data.type')
   7.181 +datauser=$(secret 'firethorn.data.user')
   7.182 +datapass=$(secret 'firethorn.data.pass')
   7.183 +
   7.184 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   7.185 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   7.186 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   7.187 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   7.188 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   7.189 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   7.190 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   7.191 +
   7.192 +tunneluser=$(secret 'ssh.tunnel.user')
   7.193 +tunnelhost=$(secret 'ssh.tunnel.host')
   7.194 +
   7.195 +admingroup=$(pwgen 20 1)	
   7.196 +adminuser=$(pwgen 20 1)
   7.197 +adminpass=$(pwgen 20 1)
   7.198 +
   7.199 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   7.200 + 
   7.201 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   7.202 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   7.203 +
   7.204 +guestgroup=friends
   7.205 +endpoint=$(secret 'endpoint')
   7.206 +
   7.207 +firethornhost=gillian
   7.208 +
   7.209 +EOF
   7.210 +
   7.211 +
   7.212 +
   7.213 +## -----------------------------------------------------
   7.214 +## Create our Firethorn properties.
   7.215 +## -----------------------------------------------------
   7.216 +
   7.217 +source "${HOME:?}/chain.properties"
   7.218 +cat > "${HOME:?}/firethorn.properties" << EOF
   7.219 +
   7.220 +firethorn.meta.type=pgsql
   7.221 +firethorn.meta.driver=org.postgresql.Driver
   7.222 +firethorn.meta.host=bethany
   7.223 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   7.224 +firethorn.meta.data=postgres
   7.225 +firethorn.meta.user=${metauser:?}
   7.226 +firethorn.meta.pass=${metapass:?}
   7.227 +
   7.228 +firethorn.user.type=mssql
   7.229 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   7.230 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   7.231 +firethorn.user.host=${userhost:?}
   7.232 +firethorn.user.data=${userdata:?}
   7.233 +firethorn.user.user=${useruser:?}
   7.234 +firethorn.user.pass=${userpass:?}
   7.235 +
   7.236 +firethorn.tapschema.resource.name=${tapschemajdbc}
   7.237 +firethorn.tapschema.database.name=${tapschemadata}
   7.238 +firethorn.tapschema.database.host=${tapschemahost}
   7.239 +firethorn.tapschema.database.port=${tapschemaport}
   7.240 +firethorn.tapschema.database.user=${tapschemauser}
   7.241 +firethorn.tapschema.database.pass=${tapschemapass}
   7.242 +
   7.243 +firethorn.admin.community=${admingroup:?}
   7.244 +firethorn.admin.user.name=${adminuser:?}
   7.245 +firethorn.admin.user.pass=${adminpass:?}
   7.246 +
   7.247 +firethorn.guest.community=${guestgroup:?}
   7.248 +
   7.249 +adql.query.delay.first=${adql_query_delay_first:?}
   7.250 +
   7.251 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   7.252 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   7.253 +
   7.254 +firethornhost=gillian
   7.255 +
   7.256 +
   7.257 +EOF
   7.258 +
   7.259 +
   7.260 +
   7.261 +## -------------------------------------------------------------------------------------------
   7.262 +## Create our compose env files.
   7.263 +## -------------------------------------------------------------------------------------------
   7.264 +
   7.265 +source "${HOME:?}/chain.properties"
   7.266 +cat > "${HOME:?}/.env" << EOF
   7.267 +
   7.268 +buildtag=${branch:?}
   7.269 +endpoint=${endpoint:?}
   7.270 +
   7.271 +
   7.272 +metadata=postgres
   7.273 +metauser=${metauser:?}
   7.274 +metapass=${metapass:?}
   7.275 +firethorn.meta.type=pgsql
   7.276 +firethorn.meta.driver=org.postgresql.Driver
   7.277 +firethorn.meta.host=bethany
   7.278 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   7.279 +firethorn.meta.data=postgres
   7.280 +firethorn.meta.user=${metauser:?}
   7.281 +firethorn.meta.pass=${metapass:?}
   7.282 +
   7.283 +
   7.284 +datadata=${datadata:?}
   7.285 +datauser=${datauser:?}
   7.286 +datapass=${datapass:?}
   7.287 +datahost=${datahost:?}
   7.288 +datacatalog=${datacatalog:?}
   7.289 +datatype=mssql
   7.290 +
   7.291 +
   7.292 +tapschemadata=${tapschemadata:?}
   7.293 +tapschemauser=${tapschemauser:?}
   7.294 +tapschemapass=${tapschemapass:?}
   7.295 +tapschemahost=${tapschemahost}
   7.296 +tapschemaport=${tapschemaport:?}
   7.297 +tapschemajdbc=${tapschemajdbc:?}
   7.298 +tapschematype=${tapschematype:?}
   7.299 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   7.300 +firethorn.tapschema.database.name=${tapschemadata:?}
   7.301 +firethorn.tapschema.database.host=${tapschemahost:?}
   7.302 +firethorn.tapschema.database.port=5432
   7.303 +firethorn.tapschema.database.user=${tapschemauser:?}
   7.304 +firethorn.tapschema.database.pass=${tapschemapass:?}
   7.305 +firethorn.tapschema.database.type=pgsql
   7.306 +
   7.307 +userhost=${userhost:?}
   7.308 +userdata=${userdata:?}
   7.309 +useruser=${useruser:?}
   7.310 +userpass=${userpass:?}
   7.311 +usertype=mssql
   7.312 +firethorn.user.type=mssql
   7.313 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   7.314 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   7.315 +firethorn.user.host=${userhost:?}
   7.316 +firethorn.user.data=${userdata:?}
   7.317 +firethorn.user.user=${useruser:?}
   7.318 +firethorn.user.pass=${userpass:?}
   7.319 +
   7.320 +
   7.321 +admingroup=${admingroup:?}
   7.322 +adminuser=${adminuser:?}
   7.323 +adminpass=${adminpass:?}
   7.324 +guestgroup=${guestgroup:?}
   7.325 +firethorn.admin.community=${admingroup:?}
   7.326 +firethorn.admin.user.name=${adminuser:?}
   7.327 +firethorn.admin.user.pass=${adminpass:?}
   7.328 +firethorn.guest.community=${guestgroup:?}
   7.329 +
   7.330 +
   7.331 +adql_query_delay_first=${adql_query_delay_first:?}
   7.332 +adql.query.delay.first=${adql_query_delay_first:?}
   7.333 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   7.334 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   7.335 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   7.336 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   7.337 +
   7.338 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   7.339 +firethornhost=gillian
   7.340 +
   7.341 +
   7.342 +EOF
   7.343 +
   7.344 +
   7.345 +source "${HOME:?}/chain.properties"
   7.346 +cat > "${HOME:?}/.tapschema_env" << EOF
   7.347 +
   7.348 +POSTGRES_DB=${tapschemadata}
   7.349 +POSTGRES_USER=${tapschemauser}
   7.350 +POSTGRES_PASSWORD=${tapschemapass}
   7.351 +
   7.352 +EOF
   7.353 +
   7.354 +
   7.355 +source "${HOME:?}/chain.properties"
   7.356 +cat > "${HOME:?}/.metadata_env" << EOF
   7.357 +
   7.358 +POSTGRES_DB=${metadata}
   7.359 +POSTGRES_USER=${metauser}
   7.360 +POSTGRES_PASSWORD=${metapass}
   7.361 +         
   7.362 +EOF
   7.363 +
   7.364 +
   7.365 +## -------------------------------------------------------------------------------------------
   7.366 +## Fetch the TAP Deployer
   7.367 +## -------------------------------------------------------------------------------------------
   7.368 +
   7.369 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   7.370 +
   7.371 +
   7.372 +
   7.373 +## ------------------------------------------------------------------------
   7.374 +## Deploy Swarm stack
   7.375 +## ------------------------------------------------------------------------
   7.376 +
   7.377 +docker stack deploy -c docker-compose.yml ft
   7.378 +
   7.379 +
   7.380 +
   7.381 +## ------------------------------------------------------------------------
   7.382 +## To remove all docker services 
   7.383 +## ------------------------------------------------------------------------
   7.384 +
   7.385 +## docker service rm $(docker service ls -q)
   7.386 +
   7.387 +
   7.388 +
   7.389 +## ------------------------------------------------------------------------
   7.390 +## To remove stack of services
   7.391 +## ------------------------------------------------------------------------
   7.392 +
   7.393 +## docker stack rm ft
   7.394 +
   7.395 +
   7.396 +[Stevedore@Lothigometh ~]$ docker ps
   7.397 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                   PORTS               NAMES
   7.398 +a6b8ce6fc1a7        firethorn/firethorn-py:2.1.35   "python3"                3 minutes ago       Up 3 minutes                                 ft_firethorn-py.1.b6ww5ws0x5hsoxzibvvmzkpg2
   7.399 +e2aa39f93f71        firethorn/ogsadai:2.1.35        "/bin/sh -c '/var/lo…"   3 minutes ago       Up 3 minutes (healthy)   8080/tcp            ft_jarmila.1.k0lafcd3mnk4omqqa6b3sfgga
   7.400 +77839250c9d9        firethorn/firethorn:2.1.35      "/bin/sh -c '/var/lo…"   3 minutes ago       Up 3 minutes (healthy)   8080/tcp            ft_gillian.1.b375fvmymiqzubvwrqi3jcp0u
   7.401 +754c7061e9a7        firethorn/postgres:2.1.35       "docker-entrypoint.s…"   4 minutes ago       Up 4 minutes             5432/tcp            ft_carolina.1.ry40mzo4vy2kipfj8s879ak95
   7.402 +166cb66e3bb1        firethorn/postgres:2.1.35       "docker-entrypoint.s…"   4 minutes ago       Up 4 minutes             5432/tcp            ft_bethany.1.jlh7gdn8oau9dcxhhmd9kafkn
   7.403 +
   7.404 +
   7.405 +
   7.406 +## ------------------------------------------------------------------------
   7.407 +## Run Docker Compose tap-deployer.yml from firethorn.py
   7.408 +## ------------------------------------------------------------------------
   7.409 +
   7.410 +docker exec -it a6b8ce6fc1a7 bash
   7.411 +
   7.412 +
   7.413 +
   7.414 +python3
   7.415 +
   7.416 +import firethorn
   7.417 +import firethorn_utils.configurator as configur
   7.418 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   7.419 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   7.420 +configurator =  configur.Configurator(ft)
   7.421 +
   7.422 +
   7.423 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   7.424 +
   7.425 + > TAP Service available at: http://gillian:8080/firethorn/tap/54/
   7.426 +
   7.427 +
   7.428 +
   7.429 +configurator.load_resources("/home/firethorn.py/firethorn/data/ssa-tap.json")
   7.430 +
   7.431 + > TAP Service available at: http://gillian:8080/firethorn/tap/57/
   7.432 +
   7.433 +
   7.434 +
   7.435 +## Update vsa json config file with VHSDR6, VVVDR5 catalogs
   7.436 +
   7.437 +nano /home/firethorn.py/firethorn/data/vsa-tap.json 
   7.438 +
   7.439 +{  
   7.440 +    
   7.441 +   ..
   7.442 +                    {
   7.443 +                        "jdbcid" : "avASDeA0",
   7.444 +                        "jdbcschema" : "dbo",
   7.445 +                        "jdbccatalog": "VVVDR5",
   7.446 +                        "adqlschema": "VVVDR5",
   7.447 +                        "metadata": {
   7.448 +                            "metadoc": "https://raw.githubusercontent.com/stvoutsin/metadata/master/metadocs/VVVDR5_TablesSchema.xml",
   7.449 +                            "catalog": "VVVDR5"
   7.450 +                        }
   7.451 +                    },
   7.452 +  .. 
   7.453 +                    {
   7.454 +                        "jdbcid" : "avASDeA0",   
   7.455 +                        "jdbcschema" : "dbo",         
   7.456 +                        "jdbccatalog": "VHSDR6",
   7.457 +                        "adqlschema": "VHSDR6",
   7.458 +                        "metadata": {
   7.459 +                            "metadoc": "https://raw.githubusercontent.com/stvoutsin/metadata/master/metadocs/VHSDR6_TablesSchema.xml",
   7.460 +                            "catalog": "VHSDR6"
   7.461 +                        }
   7.462 +                    }, 
   7.463 +  ..
   7.464 +}
   7.465 +
   7.466 +
   7.467 +
   7.468 +
   7.469 +configurator.load_resources("/home/firethorn.py/firethorn/data/vsa-tap.json")
   7.470 +
   7.471 + > TAP Service available at: http://gillian:8080/firethorn/tap/60/
   7.472 +
   7.473 +
   7.474 +configurator.load_resources("/home/firethorn.py/firethorn/data/wsa-tap.json")
   7.475 +
   7.476 +..
   7.477 +Importing JDBC Schema: UKIDSSDR10PLUS
   7.478 +ERROR:root:HTTP Error 404: 
   7.479 +Traceback (most recent call last):
   7.480 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   7.481 +    with urllib.request.urlopen( req, data) as response:
   7.482 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   7.483 +    return opener.open(url, data, timeout)
   7.484 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   7.485 +    response = meth(req, response)
   7.486 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   7.487 +    'http', request, response, code, msg, hdrs)
   7.488 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   7.489 +    return self._call_chain(*args)
   7.490 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   7.491 +    result = func(*args)
   7.492 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   7.493 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   7.494 +urllib.error.HTTPError: HTTP Error 404: 
   7.495 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   7.496 +Traceback (most recent call last):
   7.497 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   7.498 +    response_json_list = json.loads(response.text)
   7.499 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   7.500 +    return _default_decoder.decode(s)
   7.501 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   7.502 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   7.503 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   7.504 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   7.505 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   7.506 +...
   7.507 +Importing JDBC Schema: SDSSDR9
   7.508 +ERROR:root:HTTP Error 404: 
   7.509 +Traceback (most recent call last):
   7.510 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   7.511 +    with urllib.request.urlopen( req, data) as response:
   7.512 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   7.513 +    return opener.open(url, data, timeout)
   7.514 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   7.515 +    response = meth(req, response)
   7.516 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   7.517 +    'http', request, response, code, msg, hdrs)
   7.518 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   7.519 +    return self._call_chain(*args)
   7.520 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   7.521 +    result = func(*args)
   7.522 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   7.523 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   7.524 +urllib.error.HTTPError: HTTP Error 404: 
   7.525 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   7.526 +Traceback (most recent call last):
   7.527 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   7.528 +    response_json_list = json.loads(response.text)
   7.529 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   7.530 +    return _default_decoder.decode(s)
   7.531 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   7.532 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   7.533 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   7.534 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   7.535 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   7.536 +...
   7.537 +Importing JDBC Schema: SSA
   7.538 +ERROR:root:HTTP Error 404: 
   7.539 +Traceback (most recent call last):
   7.540 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   7.541 +    with urllib.request.urlopen( req, data) as response:
   7.542 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   7.543 +    return opener.open(url, data, timeout)
   7.544 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   7.545 +    response = meth(req, response)
   7.546 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   7.547 +    'http', request, response, code, msg, hdrs)
   7.548 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   7.549 +    return self._call_chain(*args)
   7.550 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   7.551 +    result = func(*args)
   7.552 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   7.553 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   7.554 +urllib.error.HTTPError: HTTP Error 404: 
   7.555 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   7.556 +Traceback (most recent call last):
   7.557 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   7.558 +    response_json_list = json.loads(response.text)
   7.559 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   7.560 +    return _default_decoder.decode(s)
   7.561 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   7.562 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   7.563 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   7.564 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   7.565 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   7.566 +..
   7.567 +Importing JDBC Schema: WISE
   7.568 + 
   7.569 +  > TAP Service available at: http://gillian:8080/firethorn/tap/63/
   7.570 +
   7.571 +
   7.572 +
   7.573 +
   7.574 +
   7.575 +## ------------------------------------------------------------------------
   7.576 +## Validate
   7.577 +## ------------------------------------------------------------------------
   7.578 +
   7.579 +java -Xmx4096m -jar stilts.jar -disk taplint tapurl=http://tap.roe.ac.uk/vsa debug=true maxtable=10
   7.580 +
   7.581 +This is STILTS taplint, 3.1-6/78f78cf (2019-05-09)
   7.582 +Static report types: ERROR(124), WARNING(48), INFO(21), SUMMARY(9), FAILURE(21)
   7.583 +
   7.584 +Section TMV: Validate table metadata against XML schema
   7.585 +I-TMV-VURL-1 Validating http://tap.roe.ac.uk/vsa/tables as tableset (http://www.ivoa.net/xml/VODataService/v1.1)
   7.586 +S-TMV-VALI-1 SAX report: warnings 0, errors 0, fatal 0
   7.587 +
   7.588 +Section TME: Check content of tables metadata from /tables
   7.589 +I-TME-TURL-1 Reading table metadata from http://tap.roe.ac.uk/vsa/tables
   7.590 +S-TME-SUMM-1 Schemas: 43, Tables: 2748, Columns: 114479, Foreign Keys: 0
   7.591 +S-TME-FLGS-1 Standard column flags: indexed: 0, primary: 0, nullable: 0
   7.592 +S-TME-FLGO-1 Other column flags: none
   7.593 +
   7.594 +Section TMS: Check content of tables metadata from TAP_SCHEMA
   7.595 +I-TMS-QSUB-1 Submitting query: SELECT COUNT(*) AS nr FROM TAP_SCHEMA.schemas
   7.596 +I-TMS-QGET-1 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+COUNT%28*%29+AS+nr+FROM+TAP_SCHEMA.schemas
   7.597 +I-TMS-QSUB-2 Submitting query: SELECT COUNT(*) AS nr FROM TAP_SCHEMA.tables
   7.598 +I-TMS-QGET-2 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+COUNT%28*%29+AS+nr+FROM+TAP_SCHEMA.tables
   7.599 +I-TMS-QSUB-3 Submitting query: SELECT COUNT(*) AS nr FROM TAP_SCHEMA.columns
   7.600 +I-TMS-QGET-3 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+COUNT%28*%29+AS+nr+FROM+TAP_SCHEMA.columns
   7.601 +I-TMS-QSUB-4 Submitting query: SELECT COUNT(*) AS nr FROM TAP_SCHEMA.keys
   7.602 +I-TMS-QGET-4 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+COUNT%28*%29+AS+nr+FROM+TAP_SCHEMA.keys
   7.603 +I-TMS-QSUB-5 Submitting query: SELECT COUNT(*) AS nr FROM TAP_SCHEMA.key_columns
   7.604 +I-TMS-QGET-5 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+COUNT%28*%29+AS+nr+FROM+TAP_SCHEMA.key_columns
   7.605 +I-TMS-QSUB-6 Submitting query: SELECT principal, indexed, std, "size" FROM TAP_SCHEMA.columns
   7.606 +I-TMS-QGET-6 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+principal%2C+indexed%2C+std%2C+%22size%22+FROM+TAP_SCHEMA.columns&MAXREC=114489
   7.607 +I-TMS-QSUB-7 Submitting query: SELECT TOP 1 * FROM TAP_SCHEMA.columns
   7.608 +I-TMS-QGET-7 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+TOP+1+*+FROM+TAP_SCHEMA.columns&MAXREC=1
   7.609 +I-TMS-QSUB-8 Submitting query: SELECT ft_column_id, std, indexed, principal, arraysize, ts_column_id, datatype, utype, ucd, unit, description, column_name, table_name FROM TAP_SCHEMA.columns
   7.610 +I-TMS-QGET-8 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+ft_column_id%2C+std%2C+indexed%2C+principal%2C+arraysize%2C+ts_column_id%2C+datatype%2C+utype%2C+ucd%2C+unit%2C+description%2C+column_name%2C+table_name+FROM+TAP_SCHEMA.columns&MAXREC=114489
   7.611 +I-TMS-QSUB-9 Submitting query: SELECT from_column, target_column, key_id FROM TAP_SCHEMA.key_columns
   7.612 +I-TMS-QGET-9 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+from_column%2C+target_column%2C+key_id+FROM+TAP_SCHEMA.key_columns&MAXREC=114489
   7.613 +S-TMS-SUMM-1 Schemas: 43, Tables: 2748, Columns: 114479, Foreign Keys: 0
   7.614 +S-TMS-FLGS-1 Standard column flags: indexed: 0, principal: 7, std: 33
   7.615 +S-TMS-QNUM-1 Successful/submitted TAP queries: 13/13
   7.616 +S-TMS-QTIM-1 Average successful query time: 4.3s
   7.617 +I-TMS-QSUB-x (4 more)
   7.618 +I-TMS-QGET-x (4 more)
   7.619 +
   7.620 +Section TMC: Compare table metadata from /tables and TAP_SCHEMA
   7.621 +
   7.622 +Section CPV: Validate capabilities against XML schema
   7.623 +I-CPV-VURL-1 Validating http://tap.roe.ac.uk/vsa/capabilities as capabilities (http://www.ivoa.net/xml/VOSICapabilities/v1.0)
   7.624 +S-CPV-VALI-1 SAX report: warnings 0, errors 0, fatal 0
   7.625 +
   7.626 +Section CAP: Check content of TAPRegExt capabilities record
   7.627 +I-CAP-CURL-1 Reading capability metadata from http://tap.roe.ac.uk/vsa/capabilities
   7.628 +
   7.629 +Section AVV: Validate availability against XML schema
   7.630 +I-AVV-VURL-1 Validating http://tap.roe.ac.uk/vsa/availability as availability (http://www.ivoa.net/xml/VOSIAvailability/v1.0)
   7.631 +S-AVV-VALI-1 SAX report: warnings 0, errors 0, fatal 0
   7.632 +
   7.633 +Section QGE: Make ADQL queries in sync GET mode
   7.634 +I-QGE-QSUB-1 Submitting query: SELECT TOP 10 surveyID FROM VIKINGDR3.ExternalSurveyTable
   7.635 +I-QGE-QGET-1 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+TOP+10+surveyID+FROM+VIKINGDR3.ExternalSurveyTable
   7.636 +I-QGE-QSUB-2 Submitting query: SELECT TOP 1 surveyID FROM VIKINGDR3.ExternalSurveyTable
   7.637 +I-QGE-QGET-2 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL-2.0&QUERY=SELECT+TOP+1+surveyID+FROM+VIKINGDR3.ExternalSurveyTable
   7.638 +I-QGE-QSUB-3 Submitting query: SELECT TOP 1 surveyID FROM VIKINGDR3.ExternalSurveyTable
   7.639 +I-QGE-QGET-3 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+TOP+1+surveyID+FROM+VIKINGDR3.ExternalSurveyTable
   7.640 +I-QGE-QSUB-4 Submitting query: SELECT TOP 8 releaseNum, productID AS taplint_c_2, V.ppErrBitsStatus, cuEventID AS taplint_c_4, multiframeID, V.programmeID AS taplint_c_6 FROM VIKINGDR3.ProgrammeFrame AS V
   7.641 +I-QGE-QGET-4 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+TOP+8+releaseNum%2C+productID+AS+taplint_c_2%2C+V.ppErrBitsStatus%2C+cuEventID+AS+taplint_c_4%2C+multiframeID%2C+V.programmeID+AS+taplint_c_6++FROM+VIKINGDR3.ProgrammeFrame+AS+V
   7.642 +I-QGE-QSUB-5 Submitting query: SELECT programmeID FROM VIKINGDR3.RequiredDiffImage
   7.643 +I-QGE-QGET-5 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=SELECT+programmeID+FROM+VIKINGDR3.RequiredDiffImage&MAXREC=0
   7.644 +I-QGE-DUFF-1 Submitting duff query: DUFF QUERY
   7.645 +I-QGE-QGET-6 Query GET URL: http://tap.roe.ac.uk/vsa/sync?REQUEST=doQuery&LANG=ADQL&QUERY=DUFF+QUERY
   7.646 +S-QGE-QNUM-1 Successful/submitted TAP queries: 5/5
   7.647 +S-QGE-QTIM-1 Average successful query time: 1.3s
   7.648 +
   7.649 +Section QPO: Make ADQL queries in sync POST mode
   7.650 +I-QPO-QSUB-1 Submitting query: SELECT TOP 10 surveyID FROM VIKINGDR3.ExternalSurveyTable
   7.651 +I-QPO-QSUB-2 Submitting query: SELECT TOP 8 releaseNum, productID AS taplint_c_2, V.ppErrBitsStatus, cuEventID AS taplint_c_4, multiframeID, V.programmeID AS taplint_c_6 FROM VIKINGDR3.ProgrammeFrame AS V
   7.652 +I-QPO-QSUB-3 Submitting query: SELECT programmeID FROM VIKINGDR3.RequiredDiffImage
   7.653 +I-QPO-DUFF-1 Submitting duff query: DUFF QUERY
   7.654 +S-QPO-QNUM-1 Successful/submitted TAP queries: 3/3
   7.655 +S-QPO-QTIM-1 Average successful query time: 1.3s
   7.656 +
   7.657 +Section QAS: Make ADQL queries in async mode
   7.658 +I-QAS-QSUB-1 Submitting query: SELECT TOP 10 surveyID FROM VIKINGDR3.ExternalSurveyTable
   7.659 +I-QAS-QJOB-1 Submitted query at http://tap.roe.ac.uk/firethorn/tap/60/async/843161
   7.660 +I-QAS-QSUB-2 Submitting query: SELECT TOP 8 releaseNum, productID AS taplint_c_2, V.ppErrBitsStatus, cuEventID AS taplint_c_4, multiframeID, V.programmeID AS taplint_c_6 FROM VIKINGDR3.ProgrammeFrame AS V
   7.661 +I-QAS-QJOB-2 Submitted query at http://tap.roe.ac.uk/firethorn/tap/60/async/843162
   7.662 +I-QAS-QSUB-3 Submitting query: SELECT programmeID FROM VIKINGDR3.RequiredDiffImage
   7.663 +I-QAS-QJOB-3 Submitted query at http://tap.roe.ac.uk/firethorn/tap/60/async/843163
   7.664 +I-QAS-DUFF-1 Submitting duff query: DUFF QUERY
   7.665 +I-QAS-QJOB-4 Submitted query at http://tap.roe.ac.uk/firethorn/tap/60/async/843164
   7.666 +S-QAS-QNUM-1 Successful/submitted TAP queries: 3/3
   7.667 +S-QAS-QTIM-1 Average successful query time: 2.2s
   7.668 +
   7.669 +Section UWS: Test asynchronous UWS/TAP behaviour
   7.670 +I-UWS-CJOB-1 Created new job http://tap.roe.ac.uk/firethorn/tap/60/async/843165
   7.671 +I-UWS-POPA-1 POSTed runId=TAPLINT-002 to http://tap.roe.ac.uk/firethorn/tap/60/async/843165/parameters
   7.672 +I-UWS-POPA-2 POSTed PHASE=ABORT to http://tap.roe.ac.uk/firethorn/tap/60/async/843165/phase
   7.673 +I-UWS-POPA-3 POSTed ACTION=DELETE to http://tap.roe.ac.uk/firethorn/tap/60/async/843165
   7.674 +I-UWS-CJOB-2 Created new job http://tap.roe.ac.uk/firethorn/tap/60/async/843166
   7.675 +I-UWS-CJOB-3 Created new job http://tap.roe.ac.uk/firethorn/tap/60/async/843167
   7.676 +I-UWS-VUWS-1 UWS job document implicitly V1.0
   7.677 +I-UWS-POPA-4 POSTed PHASE=RUN to http://tap.roe.ac.uk/firethorn/tap/60/async/843167/phase
   7.678 +
   7.679 +Section MDQ: Check table query result columns against declared metadata
   7.680 +I-MDQ-TMAX-1 Testing only 10/2748 tables
   7.681 +I-MDQ-QSUB-1 Submitting query: SELECT TOP 1 * FROM "FIRST".first08Jul16Source
   7.682 +I-MDQ-QSUB-2 Submitting query: SELECT TOP 1 * FROM "FIRST".firstSource
   7.683 +I-MDQ-QSUB-3 Submitting query: SELECT TOP 1 * FROM "FIRST".firstSource12Feb16
   7.684 +I-MDQ-QSUB-4 Submitting query: SELECT TOP 1 * FROM BestDR1."Diagnostics"
   7.685 +I-MDQ-QSUB-5 Submitting query: SELECT TOP 1 * FROM BestDR1."First"
   7.686 +I-MDQ-QSUB-6 Submitting query: SELECT TOP 1 * FROM BestDR1."Match"
   7.687 +I-MDQ-QSUB-7 Submitting query: SELECT TOP 1 * FROM BestDR1."Region"
   7.688 +I-MDQ-QSUB-8 Submitting query: SELECT TOP 1 * FROM BestDR1."Zone"
   7.689 +I-MDQ-QSUB-9 Submitting query: SELECT TOP 1 * FROM BestDR1.Best2Sector
   7.690 +S-MDQ-QNUM-1 Successful/submitted TAP queries: 10/10
   7.691 +S-MDQ-QTIM-1 Average successful query time: 1.4s
   7.692 +I-MDQ-QSUB-x (1 more)
   7.693 +
   7.694 +Section OBS: Test implementation of ObsCore Data Model
   7.695 +I-OBS-NODM-1 Table capabilities lists no ObsCore DataModel - no ObsCore tests
   7.696 +
   7.697 +Section UPL: Make queries with table uploads
   7.698 +F-UPL-NOUP-1 Table capabilities lists no upload methods - will not attempt upload tests
   7.699 +
   7.700 +Section EXA: Check content of examples document
   7.701 +I-EXA-EURL-1 Reading examples document from http://tap.roe.ac.uk/vsa/examples
   7.702 +F-EXA-EXNO-1 No examples document at http://tap.roe.ac.uk/vsa/examples
   7.703 +
   7.704 +Totals: Errors: 0; Warnings: 0; Infos: 76; Summaries: 18; Failures: 2
   7.705 +
     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/doc/notes/stv/20190903-hg-merge.txt	Wed Oct 09 16:54:28 2019 +0200
     8.3 @@ -0,0 +1,371 @@
     8.4 +#
     8.5 +# <meta:header>
     8.6 +#   <meta:licence>
     8.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     8.8 +#
     8.9 +#     This information is free software: you can redistribute it and/or modify
    8.10 +#     it under the terms of the GNU General Public License as published by
    8.11 +#     the Free Software Foundation, either version 3 of the License, or
    8.12 +#     (at your option) any later version.
    8.13 +#
    8.14 +#     This information is distributed in the hope that it will be useful,
    8.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    8.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    8.17 +#     GNU General Public License for more details.
    8.18 +#
    8.19 +#     You should have received a copy of the GNU General Public License
    8.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    8.21 +#   </meta:licence>
    8.22 +# </meta:header>
    8.23 +#
    8.24 +#
    8.25 +
    8.26 +# -----------------------------------------------------
    8.27 +# Create a new VM.
    8.28 +#[user@trop01]
    8.29 +
    8.30 +    createvm
    8.31 +
    8.32 +    >   INFO : Node name [Froeseth]
    8.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
    8.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
    8.35 +    >   INFO : Disc name [Froeseth.qcow]
    8.36 +    >   INFO : Disc size [16GiB]
    8.37 +
    8.38 +    vmname=Froeseth
    8.39 +
    8.40 +
    8.41 +
    8.42 +    getipv4()
    8.43 +        {
    8.44 +        getent hosts "${1:?}" | cut -d ' ' -f 1
    8.45 +        }
    8.46 +
    8.47 +
    8.48 +#---------------------------------------------------------------------
    8.49 +# Update the ssh keys for the VM.
    8.50 +#[user@trop01]
    8.51 +
    8.52 +    # Remove the current key.
    8.53 +    ssh-keygen -q -R "${vmname:?}"
    8.54 +
    8.55 +    # Add the host key(s) to known_hosts
    8.56 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
    8.57 +
    8.58 +    # Add the IP address key(s) to known_hosts
    8.59 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
    8.60 +
    8.61 +
    8.62 +# -----------------------------------------------------
    8.63 +# Login as Stevedore
    8.64 +#[user@trop]
    8.65 +
    8.66 +    ssh "${vmname:?}"
    8.67 +
    8.68 +
    8.69 +# -----------------------------------------------------
    8.70 +# Install our secret function.
    8.71 +#[user@virtual]
    8.72 +
    8.73 +    mkdir "${HOME:?}/bin"
    8.74 +    cat > "${HOME:?}/bin/secret" << 'EOF'
    8.75 +#!/bin/sh
    8.76 +ssh -n \
    8.77 +    'stv@shepseskaf.roe.ac.uk' \
    8.78 +    "bin/secret '${1}'"
    8.79 +EOF
    8.80 +
    8.81 +    chmod u+x "${HOME:?}/bin/secret"
    8.82 +    secret 'frog'
    8.83 +
    8.84 +
    8.85 +# -----------------------------------------------------
    8.86 +# Download our builder compose file
    8.87 +#[user@virtual]
    8.88 +
    8.89 +    wget -O builder.yml \
    8.90 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
    8.91 +
    8.92 +
    8.93 +# -----------------------------------------------------
    8.94 +# Set the target branch
    8.95 +#[user@virtual]
    8.96 +
    8.97 +    branch=2.1.34-stv-tap-reserved
    8.98 +
    8.99 +# -----------------------------------------------------
   8.100 +# Run our builder.
   8.101 +#[user@virtual]
   8.102 +
   8.103 +    export branch
   8.104 +    export secretsh=$(which secret)
   8.105 +
   8.106 +    docker-compose \
   8.107 +        --file "builder.yml" \
   8.108 +        run \
   8.109 +            builder
   8.110 +
   8.111 +    # -----------------------------------------------------
   8.112 +    # Test our secret function.
   8.113 +    #[root@builder]
   8.114 +
   8.115 +        secret 'frog'
   8.116 +
   8.117 +    # -----------------------------------------------------
   8.118 +    # Initialise our paths.
   8.119 +    #[root@builder]
   8.120 +
   8.121 +        PATH=${PATH}:/builder/bin
   8.122 +
   8.123 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   8.124 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   8.125 +
   8.126 +        export FIRETHORN_HOME
   8.127 +        export FIRETHORN_CODE
   8.128 +
   8.129 +    # -----------------------------------------------------
   8.130 +    # Checkout a copy of our source code.
   8.131 +    #[root@builder]
   8.132 +
   8.133 +        02.01-checkout.sh
   8.134 +
   8.135 +    # -----------------------------------------------------
   8.136 +    # Merge our branch into main.
   8.137 +    #[root@builder]
   8.138 +
   8.139 +        07.01-merge-start.sh
   8.140 +
   8.141 +    # -----------------------------------------------------
   8.142 +    # Update our version number.
   8.143 +    #[root@builder]
   8.144 +
   8.145 +        07.02-version-step.sh
   8.146 +
   8.147 +    # -----------------------------------------------------
   8.148 +    # Build our base images.
   8.149 +    #[root@builder]
   8.150 +
   8.151 +        04.01-buildbase.sh
   8.152 +
   8.153 +    # -----------------------------------------------------
   8.154 +    # Tag this version of builder as latest.
   8.155 +    #[root@builder]
   8.156 +
   8.157 +        source "${FIRETHORN_HOME}/merge.settings"
   8.158 +
   8.159 +        docker tag \
   8.160 +            "firethorn/builder:${newversion}" "firethorn/builder:latest"
   8.161 +
   8.162 +    # -----------------------------------------------------
   8.163 +    # Exit this version of builder.
   8.164 +    #[root@builder]
   8.165 +
   8.166 +        exit
   8.167 +
   8.168 +# -----------------------------------------------------
   8.169 +# Run a new instance of the builder.
   8.170 +#[user@virtual]
   8.171 +
   8.172 +    export branch
   8.173 +    export secretsh=$(which secret)
   8.174 +
   8.175 +    docker-compose \
   8.176 +        --file "builder.yml" \
   8.177 +        run \
   8.178 +            builder
   8.179 +
   8.180 +    # -----------------------------------------------------
   8.181 +    # Test our secret function.
   8.182 +    #[root@builder]
   8.183 +
   8.184 +        secret 'frog'
   8.185 +
   8.186 +    # -----------------------------------------------------
   8.187 +    # Initialise our paths.
   8.188 +    #[root@builder]
   8.189 +
   8.190 +		PATH=${PATH}:/builder/bin
   8.191 +
   8.192 +		: ${FIRETHORN_HOME:=/etc/firethorn}
   8.193 +		: ${FIRETHORN_CODE:=/var/local/build/firethorn}
   8.194 +
   8.195 +		export FIRETHORN_HOME
   8.196 +		export FIRETHORN_CODE
   8.197 +
   8.198 +    # -----------------------------------------------------
   8.199 +    # Build our base images.
   8.200 +    #[root@builder]
   8.201 +
   8.202 +        04.01-buildbase.sh
   8.203 +
   8.204 +    # -----------------------------------------------------
   8.205 +    # Compile our Java code.
   8.206 +    #[root@builder]
   8.207 +
   8.208 +        05.01-javamaven.sh
   8.209 +
   8.210 +    # -----------------------------------------------------
   8.211 +    # Build our Java containers.
   8.212 +    #[root@builder]
   8.213 +
   8.214 +        05.02-javadocker.sh
   8.215 +
   8.216 +    # -----------------------------------------------------
   8.217 +    # Fetch our Python code.
   8.218 +    #[root@builder]
   8.219 +
   8.220 +        echo "Fetching client source"
   8.221 +
   8.222 +        ftpysrc='/var/local/build/client'
   8.223 +        ftpyutilsfolder='/var/local/build/ftpy-utils'
   8.224 +
   8.225 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
   8.226 +        ftpyutils='https://github.com/stvoutsin/firethorn-utils.git'
   8.227 +
   8.228 +        if [  -e "${ftpysrc:?}" ]
   8.229 +        then
   8.230 +            pushd "${ftpysrc:?}"
   8.231 +
   8.232 +                echo "Updating Python code"
   8.233 +                git pull
   8.234 +
   8.235 +            popd
   8.236 +        else
   8.237 +            pushd "$(dirname ${ftpysrc:?})"
   8.238 +
   8.239 +                echo "Cloning Python code from [${ftpygit:?}]"
   8.240 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
   8.241 +                git clone "${ftpyutils:?}" "$(basename ${ftpyutilsfolder:?})"
   8.242 +
   8.243 +		docker build --tag firethorn/firethorn-utils:${buildtag} ${ftpyutilsfolder?}/docker/firethorn-utils
   8.244 +		docker build --tag firethorn/firethorn-py:${buildtag} ${ftpysrc:?}/docker/firethorn-py
   8.245 +		docker build --tag firethorn/firethorn-utils:latest  ${ftpyutilsfolder?}/docker/firethorn-utils
   8.246 +		docker build --tag firethorn/firethorn-py:latest ${ftpysrc:?}/docker/firethorn-py
   8.247 +
   8.248 +            popd
   8.249 +        fi
   8.250 +
   8.251 +  
   8.252 +	
   8.253 +
   8.254 +  # -----------------------------------------------------
   8.255 +    # Delete the embedded metadata.
   8.256 +    #[root@builder]
   8.257 +
   8.258 +        pushd "${ftpysrc:?}"
   8.259 +
   8.260 +            du -h  firethorn/meta
   8.261 +            rm -rf firethorn/meta
   8.262 +
   8.263 +        popd
   8.264 +
   8.265 +
   8.266 +# -----------------------------------------------------
   8.267 +# Run our Python tests, 20190601-02-merge-tests.txt
   8.268 +# -----------------------------------------------------
   8.269 +
   8.270 +    # -----------------------------------------------------
   8.271 +    # Update our Mercurial config.
   8.272 +    #[root@builder]
   8.273 +
   8.274 +        07.02-mercurial-user.sh
   8.275 +
   8.276 +
   8.277 +    # -----------------------------------------------------
   8.278 +    # Commit our merged code.
   8.279 +    #[root@builder]
   8.280 +
   8.281 +        07.03-merge-commit.sh
   8.282 +
   8.283 +	/var/local/build/firethorn /builder/bin
   8.284 +	Merging [2.1.34-stv-tap-reserved] into [2.1.35] (Y/n)Y
   8.285 +	Close dev branch [2.1.34-stv-tap-reserved] (Y/n)Y
   8.286 +	51 files updated, 0 files merged, 0 files removed, 0 files unresolved
   8.287 +	51 files updated, 0 files merged, 0 files removed, 0 files unresolved
   8.288 +	/builder/bin
   8.289 +
   8.290 +
   8.291 +    # -----------------------------------------------------
   8.292 +    # Push changes to Mercurial
   8.293 +    #[root@builder]
   8.294 +
   8.295 +        08.03-mercurial-push.sh
   8.296 +
   8.297 +		root@builder firethorn]#         08.03-mercurial-push.sh
   8.298 +		/var/local/build/firethorn /var/local/build/firethorn
   8.299 +		Push version [2.1.35] (Y/n)Y
   8.300 +		pushing to ssh://Stelios@data.metagrid.co.uk//var/local/wfau/projects/firethorn/code
   8.301 +		remote: sign_and_send_pubkey: no mutual signature supported
   8.302 +		remote: Stelios@data.metagrid.co.uk: Permission denied (publickey,gssapi-keyex,gssapi-with-mic).
   8.303 +		abort: no suitable response from remote hg!
   8.304 +		[root@builder firethorn]# ssh Stelios@/Stelios@data.metagrid.co.uk^C
   8.305 +		[root@builder firethorn]# ^C
   8.306 +		[root@builder firethorn]# ssh Stelios@data.metagrid.co.uk
   8.307 +		sign_and_send_pubkey: no mutual signature supported
   8.308 +		Stelios@data.metagrid.co.uk: Permission denied (publickey,gssapi-keyex,gssapi-with-mic).
   8.309 +
   8.310 +    ## Copy .ssh dsa key & config from local machine and then it works
   8.311 +
   8.312 +
   8.313 +
   8.314 +    # -----------------------------------------------------
   8.315 +    # Tag our new images
   8.316 +    #[root@builder]
   8.317 +
   8.318 +        08.01-docker-latest.sh
   8.319 +
   8.320 +
   8.321 +
   8.322 +    # -----------------------------------------------------
   8.323 +    # Push images to Docker.
   8.324 +    #[root@builder]
   8.325 +
   8.326 +    docker login \
   8.327 +        --username ${username:?} \
   8.328 +        --password ${pass:?}
   8.329 +
   8.330 +    docker push "firethorn/fedora:${newversion:?}"
   8.331 +    docker push "firethorn/java:${newversion:?}"
   8.332 +    docker push "firethorn/tomcat:${newversion:?}"
   8.333 +    docker push "firethorn/firethorn:${newversion:?}"
   8.334 +    docker push "firethorn/ogsadai:${newversion:?}"
   8.335 +
   8.336 +    docker push "firethorn/builder:${newversion:?}"
   8.337 +    docker push "firethorn/tester:${newversion:?}"
   8.338 +    docker push "firethorn/postgres:${newversion:?}"
   8.339 +    docker push "firethorn/sql-proxy:${newversion:?}"
   8.340 +
   8.341 +    docker push "firethorn/firethorn-py:${newversion:?}"
   8.342 +
   8.343 +    docker push "firethorn/fedora:latest"
   8.344 +    docker push "firethorn/java:latest"
   8.345 +    docker push "firethorn/tomcat:latest"
   8.346 +    docker push "firethorn/firethorn:latest"
   8.347 +    docker push "firethorn/ogsadai:latest"
   8.348 +
   8.349 +    docker push "firethorn/builder:latest"
   8.350 +    docker push "firethorn/tester:latest"
   8.351 +    docker push "firethorn/postgres:latest"
   8.352 +    docker push "firethorn/sql-proxy:latest"
   8.353 +
   8.354 +    docker push "firethorn/firethorn-py:latest"
   8.355 +
   8.356 +    docker push "firethorn/firethorn-utils:latest"
   8.357 +    docker push "firethorn/firethorn-utils:${newversion:?}"
   8.358 +
   8.359 +
   8.360 +    # -----------------------------------------------------
   8.361 +    # Push binaries to Maven repo.
   8.362 +    #[root@builder]
   8.363 +
   8.364 +        08.04-maven-push.sh
   8.365 +
   8.366 +	   > /builder/bin/08.04-maven-push.sh: line 29: mvnuser: parameter null or not set
   8.367 + 
   8.368 +
   8.369 +    ## Add missing value to secret file, works after that..
   8.370 +
   8.371 +
   8.372 +
   8.373 +
   8.374 +
     9.1 --- a/docker/apache/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
     9.2 +++ b/docker/apache/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
     9.3 @@ -17,7 +17,7 @@
     9.4  #
     9.5  # Based on example from the official httpd Docker container.
     9.6  # https://github.com/docker-library/httpd/blob/master/2.4/Dockerfile
     9.7 -FROM firethorn/fedora:2.1.33
     9.8 +FROM firethorn/fedora:2.1.35
     9.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
    9.10  
    9.11  #
    10.1 --- a/docker/builder/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    10.2 +++ b/docker/builder/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    10.3 @@ -15,7 +15,7 @@
    10.4  # You should have received a copy of the GNU General Public License
    10.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    10.6  
    10.7 -FROM firethorn/java:2.1.33
    10.8 +FROM firethorn/java:2.1.35
    10.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   10.10  
   10.11  #
    11.1 --- a/docker/compose/client/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    11.2 +++ b/docker/compose/client/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    11.3 @@ -15,7 +15,7 @@
    11.4  # You should have received a copy of the GNU General Public License
    11.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    11.6  
    11.7 -FROM firethorn/tester:2.1.33
    11.8 +FROM firethorn/tester:2.1.35
    11.9  LABEL MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   11.10  
   11.11  ENV pysrc=/home/firethorn-py
    12.1 --- a/docker/compose/tests/distictella/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    12.2 +++ b/docker/compose/tests/distictella/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    12.3 @@ -15,7 +15,7 @@
    12.4  # You should have received a copy of the GNU General Public License
    12.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    12.6  
    12.7 -FROM firethorn/fedora:2.1.33
    12.8 +FROM firethorn/fedora:2.1.35
    12.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   12.10  
   12.11  RUN dnf -y install \
    13.1 --- a/docker/compose/tests/distictella/firethorn-py/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    13.2 +++ b/docker/compose/tests/distictella/firethorn-py/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    13.3 @@ -15,7 +15,7 @@
    13.4  # You should have received a copy of the GNU General Public License
    13.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    13.6  
    13.7 -FROM firethorn/fedora:2.1.33
    13.8 +FROM firethorn/fedora:2.1.35
    13.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   13.10  
   13.11  RUN dnf -y install \
    14.1 --- a/docker/java/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    14.2 +++ b/docker/java/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    14.3 @@ -15,7 +15,7 @@
    14.4  # You should have received a copy of the GNU General Public License
    14.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    14.6  
    14.7 -FROM firethorn/fedora:2.1.33
    14.8 +FROM firethorn/fedora:2.1.35
    14.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   14.10  
   14.11  ENV JAVA_VERSION=12.0.0.33
    15.1 --- a/docker/python/3.4.2/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    15.2 +++ b/docker/python/3.4.2/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    15.3 @@ -1,4 +1,4 @@
    15.4 -FROM firethorn/ubuntu:2.1.33
    15.5 +FROM firethorn/ubuntu:2.1.35
    15.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    15.7  
    15.8  #
    16.1 --- a/docker/pythonlibs/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    16.2 +++ b/docker/pythonlibs/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    16.3 @@ -1,4 +1,4 @@
    16.4 -FROM firethorn/python:2.1.33
    16.5 +FROM firethorn/python:2.1.35
    16.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    16.7  
    16.8  # Install Python Libs
    17.1 --- a/docker/sql-proxy/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    17.2 +++ b/docker/sql-proxy/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    17.3 @@ -15,7 +15,7 @@
    17.4  # You should have received a copy of the GNU General Public License
    17.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17.6  
    17.7 -FROM firethorn/fedora:2.1.33
    17.8 +FROM firethorn/fedora:2.1.35
    17.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   17.10  
   17.11  EXPOSE 1433
    18.1 --- a/docker/sql-tunnel/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    18.2 +++ b/docker/sql-tunnel/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    18.3 @@ -15,7 +15,7 @@
    18.4  # You should have received a copy of the GNU General Public License
    18.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    18.6  
    18.7 -FROM firethorn/fedora:2.1.33
    18.8 +FROM firethorn/fedora:2.1.35
    18.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   18.10  
   18.11  EXPOSE 1433
    19.1 --- a/docker/ssh-client/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    19.2 +++ b/docker/ssh-client/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    19.3 @@ -15,7 +15,7 @@
    19.4  # You should have received a copy of the GNU General Public License
    19.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    19.6  
    19.7 -FROM firethorn/fedora:2.1.33
    19.8 +FROM firethorn/fedora:2.1.35
    19.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   19.10  
   19.11  RUN dnf -y install openssh-clients
    20.1 --- a/docker/tester/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    20.2 +++ b/docker/tester/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    20.3 @@ -15,7 +15,7 @@
    20.4  # You should have received a copy of the GNU General Public License
    20.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    20.6  
    20.7 -FROM firethorn/fedora:2.1.33
    20.8 +FROM firethorn/fedora:2.1.35
    20.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   20.10  
   20.11  RUN dnf install -y jq
    21.1 --- a/docker/tomcat/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    21.2 +++ b/docker/tomcat/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    21.3 @@ -15,7 +15,7 @@
    21.4  # You should have received a copy of the GNU General Public License
    21.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    21.6  
    21.7 -FROM firethorn/java:2.1.33
    21.8 +FROM firethorn/java:2.1.35
    21.9  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
   21.10  
   21.11  # https://tomcat.apache.org/whichversion.html
    22.1 --- a/firethorn-admin/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    22.2 +++ b/firethorn-admin/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    22.3 @@ -32,7 +32,7 @@
    22.4      <parent>
    22.5          <groupId>uk.ac.roe.wfau</groupId>
    22.6          <artifactId>firethorn</artifactId>
    22.7 -        <version project='firethorn'>2.1.33</version>
    22.8 +        <version project='firethorn'>2.1.35</version>
    22.9      </parent>
   22.10  
   22.11      <artifactId>firethorn-admin-webapp</artifactId>
    23.1 --- a/firethorn-admin/src/main/docker/firethorn/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    23.2 +++ b/firethorn-admin/src/main/docker/firethorn/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    23.3 @@ -15,7 +15,7 @@
    23.4  # You should have received a copy of the GNU General Public License
    23.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    23.6  
    23.7 -FROM firethorn/tomcat:2.1.33
    23.8 +FROM firethorn/tomcat:2.1.35
    23.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   23.10  
   23.11  #
    24.1 --- a/firethorn-annotations/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    24.2 +++ b/firethorn-annotations/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    24.3 @@ -32,7 +32,7 @@
    24.4      <parent>
    24.5          <groupId>uk.ac.roe.wfau</groupId>
    24.6          <artifactId>firethorn</artifactId>
    24.7 -        <version project='firethorn'>2.1.33</version>
    24.8 +        <version project='firethorn'>2.1.35</version>
    24.9      </parent>
   24.10  
   24.11      <artifactId>firethorn-annotations</artifactId>
    25.1 --- a/firethorn-cds/adql/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    25.2 +++ b/firethorn-cds/adql/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    25.3 @@ -32,7 +32,7 @@
    25.4      <parent>
    25.5          <groupId>uk.ac.roe.wfau</groupId>
    25.6          <artifactId>firethorn-cds</artifactId>
    25.7 -		<version project='firethorn'>2.1.33</version>
    25.8 +		<version project='firethorn'>2.1.35</version>
    25.9  	</parent>
   25.10  
   25.11      <artifactId>firethorn-cds-adql</artifactId>
    26.1 --- a/firethorn-cds/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    26.2 +++ b/firethorn-cds/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    26.3 @@ -32,7 +32,7 @@
    26.4      <parent>
    26.5          <groupId>uk.ac.roe.wfau</groupId>
    26.6          <artifactId>firethorn</artifactId>
    26.7 -        <version project='firethorn'>2.1.33</version>
    26.8 +        <version project='firethorn'>2.1.35</version>
    26.9      </parent>
   26.10  
   26.11      <artifactId>firethorn-cds</artifactId>
    27.1 --- a/firethorn-core/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    27.2 +++ b/firethorn-core/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    27.3 @@ -32,7 +32,7 @@
    27.4      <parent>
    27.5          <groupId>uk.ac.roe.wfau</groupId>
    27.6          <artifactId>firethorn</artifactId>
    27.7 -        <version project='firethorn'>2.1.33</version>
    27.8 +        <version project='firethorn'>2.1.35</version>
    27.9  	</parent>
   27.10  
   27.11      <artifactId>firethorn-core</artifactId>
    28.1 --- a/firethorn-core/src/main/java/uk/ac/roe/wfau/firethorn/adql/util/AdqlNameModifier.java	Wed Oct 09 16:53:55 2019 +0200
    28.2 +++ b/firethorn-core/src/main/java/uk/ac/roe/wfau/firethorn/adql/util/AdqlNameModifier.java	Wed Oct 09 16:54:28 2019 +0200
    28.3 @@ -75,7 +75,8 @@
    28.4              "area",
    28.5              "key",
    28.6              "count",
    28.7 -            "when"
    28.8 +            "when",
    28.9 +            "overlaps"
   28.10              );
   28.11      
   28.12  
    29.1 --- a/firethorn-jdbc/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    29.2 +++ b/firethorn-jdbc/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    29.3 @@ -32,7 +32,7 @@
    29.4      <parent>
    29.5          <groupId>uk.ac.roe.wfau</groupId>
    29.6          <artifactId>firethorn</artifactId>
    29.7 -        <version project='firethorn'>2.1.33</version>
    29.8 +        <version project='firethorn'>2.1.35</version>
    29.9      </parent>
   29.10  
   29.11      <artifactId>firethorn-jdbc</artifactId>
    30.1 --- a/firethorn-logging/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    30.2 +++ b/firethorn-logging/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    30.3 @@ -32,7 +32,7 @@
    30.4      <parent>
    30.5          <groupId>uk.ac.roe.wfau</groupId>
    30.6          <artifactId>firethorn</artifactId>
    30.7 -        <version project='firethorn'>2.1.33</version>
    30.8 +        <version project='firethorn'>2.1.35</version>
    30.9      </parent>
   30.10  
   30.11      <artifactId>firethorn-logging</artifactId>
    31.1 --- a/firethorn-ogsadai/activity/client/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    31.2 +++ b/firethorn-ogsadai/activity/client/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    31.3 @@ -32,7 +32,7 @@
    31.4      <parent>
    31.5          <groupId>uk.ac.roe.wfau</groupId>
    31.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    31.7 -        <version project='firethorn'>2.1.33</version>
    31.8 +        <version project='firethorn'>2.1.35</version>
    31.9  	</parent>
   31.10  
   31.11      <artifactId>firethorn-ogsadai-activity-client</artifactId>
    32.1 --- a/firethorn-ogsadai/activity/common/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    32.2 +++ b/firethorn-ogsadai/activity/common/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    32.3 @@ -32,7 +32,7 @@
    32.4      <parent>
    32.5          <groupId>uk.ac.roe.wfau</groupId>
    32.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    32.7 -        <version project='firethorn'>2.1.33</version>
    32.8 +        <version project='firethorn'>2.1.35</version>
    32.9      </parent>
   32.10  
   32.11      <artifactId>firethorn-ogsadai-activity-common</artifactId>
    33.1 --- a/firethorn-ogsadai/activity/context/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    33.2 +++ b/firethorn-ogsadai/activity/context/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    33.3 @@ -32,7 +32,7 @@
    33.4      <parent>
    33.5          <groupId>uk.ac.roe.wfau</groupId>
    33.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    33.7 -        <version project='firethorn'>2.1.33</version>
    33.8 +        <version project='firethorn'>2.1.35</version>
    33.9  	</parent>
   33.10  
   33.11      <artifactId>firethorn-ogsadai-activity-context</artifactId>
    34.1 --- a/firethorn-ogsadai/activity/context/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/context/RequestContextFactory.java	Wed Oct 09 16:53:55 2019 +0200
    34.2 +++ b/firethorn-ogsadai/activity/context/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/context/RequestContextFactory.java	Wed Oct 09 16:54:28 2019 +0200
    34.3 @@ -64,8 +64,8 @@
    34.4          log.debug("remote [" + remote + "]");
    34.5  
    34.6          return new RequestContext()
    34.7 -			{
    34.8 -			private EndpointBuilder builder = new EndpointBuilder()
    34.9 +	    {
   34.10 +	    private EndpointBuilder builder = new EndpointBuilder()
   34.11                  {
   34.12                  private String protocol = DEFAULT_PROTOCOL;
   34.13                  @Override
   34.14 @@ -79,7 +79,7 @@
   34.15                      this.protocol = protocol;
   34.16                      }
   34.17  
   34.18 -                private String host = (remote != null) ? remote : DEFAULT_HOST ;
   34.19 +                private String host =(System.getenv("firethornhost") != null) ? System.getenv("firethornhost") : (remote != null) ? remote : DEFAULT_HOST ;
   34.20                  @Override
   34.21                  public String host()
   34.22                      {
    35.1 --- a/firethorn-ogsadai/activity/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    35.2 +++ b/firethorn-ogsadai/activity/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    35.3 @@ -37,7 +37,7 @@
    35.4      <parent>
    35.5          <groupId>uk.ac.roe.wfau</groupId>
    35.6          <artifactId>firethorn-ogsadai</artifactId>
    35.7 -        <version project='firethorn'>2.1.33</version>
    35.8 +        <version project='firethorn'>2.1.35</version>
    35.9      </parent>
   35.10  
   35.11      <artifactId>firethorn-ogsadai-activity</artifactId>
    36.1 --- a/firethorn-ogsadai/activity/server/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    36.2 +++ b/firethorn-ogsadai/activity/server/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    36.3 @@ -32,7 +32,7 @@
    36.4      <parent>
    36.5          <groupId>uk.ac.roe.wfau</groupId>
    36.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    36.7 -        <version project='firethorn'>2.1.33</version>
    36.8 +        <version project='firethorn'>2.1.35</version>
    36.9      </parent>
   36.10  
   36.11      <artifactId>firethorn-ogsadai-activity-server</artifactId>
    37.1 --- a/firethorn-ogsadai/astro/client/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    37.2 +++ b/firethorn-ogsadai/astro/client/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    37.3 @@ -32,7 +32,7 @@
    37.4      <parent>
    37.5          <groupId>uk.ac.roe.wfau</groupId>
    37.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    37.7 -        <version project='firethorn'>2.1.33</version>
    37.8 +        <version project='firethorn'>2.1.35</version>
    37.9      </parent>
   37.10  
   37.11      <artifactId>firethorn-ogsadai-astro-client</artifactId>
    38.1 --- a/firethorn-ogsadai/astro/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    38.2 +++ b/firethorn-ogsadai/astro/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    38.3 @@ -32,7 +32,7 @@
    38.4      <parent>
    38.5          <groupId>uk.ac.roe.wfau</groupId>
    38.6          <artifactId>firethorn-ogsadai</artifactId>
    38.7 -        <version project='firethorn'>2.1.33</version>
    38.8 +        <version project='firethorn'>2.1.35</version>
    38.9      </parent>
   38.10  
   38.11      <artifactId>firethorn-ogsadai-astro</artifactId>
    39.1 --- a/firethorn-ogsadai/astro/server/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    39.2 +++ b/firethorn-ogsadai/astro/server/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    39.3 @@ -32,7 +32,7 @@
    39.4      <parent>
    39.5          <groupId>uk.ac.roe.wfau</groupId>
    39.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    39.7 -        <version project='firethorn'>2.1.33</version>
    39.8 +        <version project='firethorn'>2.1.35</version>
    39.9      </parent>
   39.10  
   39.11      <artifactId>firethorn-ogsadai-astro-server</artifactId>
    40.1 --- a/firethorn-ogsadai/depends/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    40.2 +++ b/firethorn-ogsadai/depends/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    40.3 @@ -32,7 +32,7 @@
    40.4      <parent>
    40.5          <groupId>uk.ac.roe.wfau</groupId>
    40.6          <artifactId>firethorn-ogsadai</artifactId>
    40.7 -        <version project='firethorn'>2.1.33</version>
    40.8 +        <version project='firethorn'>2.1.35</version>
    40.9      </parent>
   40.10  
   40.11      <artifactId>firethorn-ogsadai-depends</artifactId>
    41.1 --- a/firethorn-ogsadai/distributed/client/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    41.2 +++ b/firethorn-ogsadai/distributed/client/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    41.3 @@ -32,7 +32,7 @@
    41.4      <parent>
    41.5          <groupId>uk.ac.roe.wfau</groupId>
    41.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    41.7 -        <version project='firethorn'>2.1.33</version>
    41.8 +        <version project='firethorn'>2.1.35</version>
    41.9      </parent>
   41.10  
   41.11      <artifactId>firethorn-ogsadai-dqp-client</artifactId>
    42.1 --- a/firethorn-ogsadai/distributed/common/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    42.2 +++ b/firethorn-ogsadai/distributed/common/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    42.3 @@ -32,7 +32,7 @@
    42.4      <parent>
    42.5          <groupId>uk.ac.roe.wfau</groupId>
    42.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    42.7 -        <version project='firethorn'>2.1.33</version>
    42.8 +        <version project='firethorn'>2.1.35</version>
    42.9      </parent>
   42.10  
   42.11      <artifactId>firethorn-ogsadai-dqp-common</artifactId>
    43.1 --- a/firethorn-ogsadai/distributed/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    43.2 +++ b/firethorn-ogsadai/distributed/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    43.3 @@ -32,7 +32,7 @@
    43.4      <parent>
    43.5          <groupId>uk.ac.roe.wfau</groupId>
    43.6          <artifactId>firethorn-ogsadai</artifactId>
    43.7 -        <version project='firethorn'>2.1.33</version>
    43.8 +        <version project='firethorn'>2.1.35</version>
    43.9      </parent>
   43.10  
   43.11      <artifactId>firethorn-ogsadai-dqp</artifactId>
    44.1 --- a/firethorn-ogsadai/distributed/server/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    44.2 +++ b/firethorn-ogsadai/distributed/server/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    44.3 @@ -32,7 +32,7 @@
    44.4      <parent>
    44.5          <groupId>uk.ac.roe.wfau</groupId>
    44.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    44.7 -        <version project='firethorn'>2.1.33</version>
    44.8 +        <version project='firethorn'>2.1.35</version>
    44.9      </parent>
   44.10  
   44.11      <artifactId>firethorn-ogsadai-dqp-server</artifactId>
    45.1 --- a/firethorn-ogsadai/firethorn/client/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    45.2 +++ b/firethorn-ogsadai/firethorn/client/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    45.3 @@ -32,7 +32,7 @@
    45.4      <parent>
    45.5          <groupId>uk.ac.roe.wfau</groupId>
    45.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    45.7 -        <version project='firethorn'>2.1.33</version>
    45.8 +        <version project='firethorn'>2.1.35</version>
    45.9      </parent>
   45.10  
   45.11      <artifactId>firethorn-ogsadai-firethorn-client</artifactId>
    46.1 --- a/firethorn-ogsadai/firethorn/common/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    46.2 +++ b/firethorn-ogsadai/firethorn/common/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    46.3 @@ -32,7 +32,7 @@
    46.4      <parent>
    46.5          <groupId>uk.ac.roe.wfau</groupId>
    46.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    46.7 -        <version project='firethorn'>2.1.33</version>
    46.8 +        <version project='firethorn'>2.1.35</version>
    46.9      </parent>
   46.10  
   46.11      <artifactId>firethorn-ogsadai-firethorn-common</artifactId>
    47.1 --- a/firethorn-ogsadai/firethorn/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    47.2 +++ b/firethorn-ogsadai/firethorn/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    47.3 @@ -32,7 +32,7 @@
    47.4      <parent>
    47.5          <groupId>uk.ac.roe.wfau</groupId>
    47.6          <artifactId>firethorn-ogsadai</artifactId>
    47.7 -        <version project='firethorn'>2.1.33</version>
    47.8 +        <version project='firethorn'>2.1.35</version>
    47.9      </parent>
   47.10  
   47.11      <artifactId>firethorn-ogsadai-firethorn</artifactId>
    48.1 --- a/firethorn-ogsadai/firethorn/server/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    48.2 +++ b/firethorn-ogsadai/firethorn/server/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    48.3 @@ -32,7 +32,7 @@
    48.4      <parent>
    48.5          <groupId>uk.ac.roe.wfau</groupId>
    48.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    48.7 -        <version project='firethorn'>2.1.33</version>
    48.8 +        <version project='firethorn'>2.1.35</version>
    48.9      </parent>
   48.10  
   48.11      <artifactId>firethorn-ogsadai-firethorn-server</artifactId>
    49.1 --- a/firethorn-ogsadai/metadata/client/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    49.2 +++ b/firethorn-ogsadai/metadata/client/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    49.3 @@ -32,7 +32,7 @@
    49.4      <parent>
    49.5          <groupId>uk.ac.roe.wfau</groupId>
    49.6          <artifactId>firethorn-ogsadai-metadata</artifactId>
    49.7 -        <version project='firethorn'>2.1.33</version>
    49.8 +        <version project='firethorn'>2.1.35</version>
    49.9      </parent>
   49.10  
   49.11      <artifactId>firethorn-ogsadai-metadata-client</artifactId>
    50.1 --- a/firethorn-ogsadai/metadata/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    50.2 +++ b/firethorn-ogsadai/metadata/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    50.3 @@ -32,7 +32,7 @@
    50.4      <parent>
    50.5          <groupId>uk.ac.roe.wfau</groupId>
    50.6          <artifactId>firethorn-ogsadai</artifactId>
    50.7 -        <version project='firethorn'>2.1.33</version>
    50.8 +        <version project='firethorn'>2.1.35</version>
    50.9      </parent>
   50.10  
   50.11      <artifactId>firethorn-ogsadai-metadata</artifactId>
    51.1 --- a/firethorn-ogsadai/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    51.2 +++ b/firethorn-ogsadai/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    51.3 @@ -37,7 +37,7 @@
    51.4      <parent>
    51.5          <groupId>uk.ac.roe.wfau</groupId>
    51.6          <artifactId>firethorn</artifactId>
    51.7 -        <version project='firethorn'>2.1.33</version>
    51.8 +        <version project='firethorn'>2.1.35</version>
    51.9      </parent>
   51.10  
   51.11      <artifactId>firethorn-ogsadai</artifactId>
    52.1 --- a/firethorn-ogsadai/webapp/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    52.2 +++ b/firethorn-ogsadai/webapp/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    52.3 @@ -32,7 +32,7 @@
    52.4      <parent>
    52.5          <groupId>uk.ac.roe.wfau</groupId>
    52.6          <artifactId>firethorn-ogsadai</artifactId>
    52.7 -        <version project='firethorn'>2.1.33</version>
    52.8 +        <version project='firethorn'>2.1.35</version>
    52.9      </parent>
   52.10  
   52.11      <artifactId>firethorn-ogsadai-webapp</artifactId>
    53.1 --- a/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    53.2 +++ b/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    53.3 @@ -15,7 +15,7 @@
    53.4  # You should have received a copy of the GNU General Public License
    53.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    53.6  
    53.7 -FROM firethorn/tomcat:2.1.33
    53.8 +FROM firethorn/tomcat:2.1.35
    53.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   53.10  
   53.11  #
    54.1 --- a/firethorn-spring/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    54.2 +++ b/firethorn-spring/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    54.3 @@ -32,7 +32,7 @@
    54.4      <parent>
    54.5          <groupId>uk.ac.roe.wfau</groupId>
    54.6          <artifactId>firethorn</artifactId>
    54.7 -        <version project='firethorn'>2.1.33</version>
    54.8 +        <version project='firethorn'>2.1.35</version>
    54.9      </parent>
   54.10  
   54.11      <artifactId>firethorn-spring</artifactId>
    55.1 --- a/firethorn-stil/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    55.2 +++ b/firethorn-stil/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    55.3 @@ -32,7 +32,7 @@
    55.4      <parent>
    55.5          <groupId>uk.ac.roe.wfau</groupId>
    55.6          <artifactId>firethorn</artifactId>
    55.7 -        <version project='firethorn'>2.1.33</version>
    55.8 +        <version project='firethorn'>2.1.35</version>
    55.9      </parent>
   55.10  
   55.11      <artifactId>firethorn-stil</artifactId>
    56.1 --- a/firethorn-testing/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    56.2 +++ b/firethorn-testing/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    56.3 @@ -32,7 +32,7 @@
    56.4      <parent>
    56.5          <groupId>uk.ac.roe.wfau</groupId>
    56.6          <artifactId>firethorn</artifactId>
    56.7 -        <version project='firethorn'>2.1.33</version>
    56.8 +        <version project='firethorn'>2.1.35</version>
    56.9      </parent>
   56.10  
   56.11      <artifactId>firethorn-testing</artifactId>
    57.1 --- a/firethorn-webapp/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    57.2 +++ b/firethorn-webapp/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    57.3 @@ -32,7 +32,7 @@
    57.4      <parent>
    57.5          <groupId>uk.ac.roe.wfau</groupId>
    57.6          <artifactId>firethorn</artifactId>
    57.7 -        <version project='firethorn'>2.1.33</version>
    57.8 +        <version project='firethorn'>2.1.35</version>
    57.9      </parent>
   57.10  
   57.11      <artifactId>firethorn-webapp</artifactId>
    58.1 --- a/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    58.2 +++ b/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    58.3 @@ -15,7 +15,7 @@
    58.4  # You should have received a copy of the GNU General Public License
    58.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    58.6  
    58.7 -FROM firethorn/tomcat:2.1.33
    58.8 +FROM firethorn/tomcat:2.1.35
    58.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   58.10  
   58.11  #
    59.1 --- a/integration/005/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    59.2 +++ b/integration/005/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    59.3 @@ -32,7 +32,7 @@
    59.4      <parent>
    59.5          <groupId>uk.ac.roe.wfau</groupId>
    59.6          <artifactId>firethorn</artifactId>
    59.7 -        <version project='firethorn'>2.1.33</version>
    59.8 +        <version project='firethorn'>2.1.35</version>
    59.9      </parent>
   59.10  
   59.11      <artifactId>firethorn-integration-tests</artifactId>
    60.1 --- a/integration/tester/Dockerfile	Wed Oct 09 16:53:55 2019 +0200
    60.2 +++ b/integration/tester/Dockerfile	Wed Oct 09 16:54:28 2019 +0200
    60.3 @@ -15,7 +15,7 @@
    60.4  # You should have received a copy of the GNU General Public License
    60.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    60.6  
    60.7 -FROM firethorn/fedora:2.1.33
    60.8 +FROM firethorn/fedora:2.1.35
    60.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   60.10  
   60.11  RUN dnf -y install perl
    61.1 --- a/pom.xml	Wed Oct 09 16:53:55 2019 +0200
    61.2 +++ b/pom.xml	Wed Oct 09 16:54:28 2019 +0200
    61.3 @@ -39,7 +39,7 @@
    61.4          | Maven project versions
    61.5          | http://www.sonatype.com/books/mvnref-book/reference/pom-relationships-sect-pom-syntax.html
    61.6          +-->
    61.7 -    <version project='firethorn'>2.1.33</version>
    61.8 +    <version project='firethorn'>2.1.35</version>
    61.9  
   61.10      <prerequisites>
   61.11          <maven>3.3.1</maven>