firethorn

changeset 4272:81afd7fe02ea

Deployment notes and test notes
author Stelios <stv@roe.ac.uk>
date Sun Jul 21 01:29:40 2019 +0300 (2019-07-21)
parents 50ac2197aff1
children 47fff5723895
files doc/notes/stv/20190720-Firethorn-Swarm-Deploy.txt doc/notes/stv/20190720-Test-Firethorn-Unresponsive.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190720-Firethorn-Swarm-Deploy.txt	Sun Jul 21 01:29:40 2019 +0300
     1.3 @@ -0,0 +1,648 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#  
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +############################################################################
    1.27 +##### Notes on Deploying WFAU TAP Services on trop02 with Docker Swarm #####
    1.28 +############################################################################
    1.29 +
    1.30 +## Aim of 2.1.34 version is an autorecovering TAP Service, using Swarm, addressing a Swarm Networking issue
    1.31 +## OGSADAI cannot get the original IP address of Firethorn in docker swarm mode 
    1.32 +## https://github.com/moby/moby/issues/25526
    1.33 +
    1.34 +stv@trop02:~$ createvm
    1.35 +
    1.36 +
    1.37 +	## INFO : Node name [Lothigometh]
    1.38 +	## INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    1.39 +	## INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    1.40 +	## INFO : Disc name [Lothigometh.qcow]
    1.41 +	## INFO : Disc size [32GiB]
    1.42 +
    1.43 +
    1.44 +
    1.45 +
    1.46 +ssh Stevedore@Lothigometh
    1.47 +
    1.48 +
    1.49 +
    1.50 +## ------------------------------------------------------------------------------------------
    1.51 +## Initialize Swarm with VM's address
    1.52 +## ------------------------------------------------------------------------------------------
    1.53 +
    1.54 +[Stevedore@Lothigometh ~]$ ifconfig
    1.55 +
    1.56 +       ## ..  192.168.202.9  ..
    1.57 +
    1.58 +
    1.59 +
    1.60 +docker swarm init --advertise-addr 192.168.202.9
    1.61 +
    1.62 +
    1.63 +	# To add a worker to this swarm, run the following command:
    1.64 +
    1.65 +	#     docker swarm join --token SWMTKN-1-5e6vdte3jggnih4t2fu5vrk4o0p2twg5mcjy7bi29s8lrv1uxs-6b09cgappan7fv9myme1pi39w 192.168.202.9:2377
    1.66 +
    1.67 +	# To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    1.68 +
    1.69 +
    1.70 +
    1.71 +## ------------------------------------------------------------------------------------------
    1.72 +## Configure Firewall for Docker Swarm
    1.73 +## https://www.digitalocean.com/community/tutorials/how-to-configure-the-linux-firewall-for-docker-swarm-on-centos-7
    1.74 +## ------------------------------------------------------------------------------------------
    1.75 +
    1.76 +
    1.77 +
    1.78 +    ## Fix Firewall issue
    1.79 +    sudo su
    1.80 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.81 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.82 +	    firewall-cmd --add-port=7946/tcp --permanent
    1.83 +	    firewall-cmd --add-port=4789/tcp --permanent
    1.84 +	    firewall-cmd --add-port=4789/udp --permanent
    1.85 +	    firewall-cmd --reload
    1.86 +    exit
    1.87 +
    1.88 +
    1.89 +## ------------------------------------------------------------------------------------------
    1.90 +## Set the target branch and builtag
    1.91 +## ------------------------------------------------------------------------------------------
    1.92 +
    1.93 +
    1.94 +    export buildtag=2.1.34
    1.95 +    export branch=default
    1.96 +
    1.97 +
    1.98 +
    1.99 +## ------------------------------------------------------------------------------------------
   1.100 +## Add secret function.
   1.101 +## ------------------------------------------------------------------------------------------
   1.102 +
   1.103 +    secrethost='stv@shepseskaf.roe.ac.uk'
   1.104 +    secretfile='${HOME:?}/secret.store.20181212'
   1.105 +
   1.106 +    secret()
   1.107 +    {
   1.108 +        local key=${1:?}
   1.109 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
   1.110 +    }
   1.111 +
   1.112 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
   1.113 +
   1.114 +
   1.115 +# ssh: connect to host shepseskaf.roe.ac.uk port 22: No route to host
   1.116 +# Shepseskaf is not accessible, looks like its down (todo: email Mark to see what is going on)
   1.117 +
   1.118 +
   1.119 +# For now, use properties from the last deploy 
   1.120 +# Stevedore@Araybwyn (/home/Stevedore/chain.properties)
   1.121 +# The example below is what is needed when the above secret function can access the server where the secrets are stored
   1.122 +
   1.123 +
   1.124 +
   1.125 +
   1.126 +## -------------------------------------------------------------------------------------------
   1.127 +## Create our chain.properties file
   1.128 +## -------------------------------------------------------------------------------------------
   1.129 +
   1.130 +cat > "${HOME:?}/chain.properties" << EOF
   1.131 +
   1.132 +metadata=postgres
   1.133 +metauser=$(pwgen 20 1)
   1.134 +metapass=$(pwgen 20 1) 
   1.135 +
   1.136 +userhost=$(secret 'firethorn.user.host')
   1.137 +userdata=$(secret 'firethorn.user.data')
   1.138 +useruser=$(secret 'firethorn.user.user')
   1.139 +userpass=$(secret 'firethorn.user.pass')
   1.140 +usertype=mssql
   1.141 +
   1.142 +datahost=$(secret 'firethorn.data.host')
   1.143 +datadata=$(secret 'firethorn.data.data')
   1.144 +datacatalog=$(secret 'firethorn.data.catalog')
   1.145 +datatype=$(secret 'firethorn.data.type')
   1.146 +datauser=$(secret 'firethorn.data.user')
   1.147 +datapass=$(secret 'firethorn.data.pass')
   1.148 +
   1.149 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   1.150 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   1.151 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   1.152 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   1.153 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   1.154 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   1.155 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   1.156 +
   1.157 +tunneluser=$(secret 'ssh.tunnel.user')
   1.158 +tunnelhost=$(secret 'ssh.tunnel.host')
   1.159 +
   1.160 +admingroup=$(pwgen 20 1)	
   1.161 +adminuser=$(pwgen 20 1)
   1.162 +adminpass=$(pwgen 20 1)
   1.163 +
   1.164 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   1.165 + 
   1.166 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   1.167 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   1.168 +
   1.169 +guestgroup=friends
   1.170 +endpoint=$(secret 'endpoint')
   1.171 +
   1.172 +firethornhost=gillian
   1.173 +
   1.174 +EOF
   1.175 +
   1.176 +
   1.177 +
   1.178 +## ------------------------------------------------------------------------------------------
   1.179 +## Create our Firethorn properties.
   1.180 +## ------------------------------------------------------------------------------------------
   1.181 +
   1.182 +source "${HOME:?}/chain.properties"
   1.183 +cat > "${HOME:?}/firethorn.properties" << EOF
   1.184 +
   1.185 +firethorn.meta.type=pgsql
   1.186 +firethorn.meta.driver=org.postgresql.Driver
   1.187 +firethorn.meta.host=bethany
   1.188 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   1.189 +firethorn.meta.data=postgres
   1.190 +firethorn.meta.user=${metauser:?}
   1.191 +firethorn.meta.pass=${metapass:?}
   1.192 +
   1.193 +firethorn.user.type=mssql
   1.194 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.195 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.196 +firethorn.user.host=${userhost:?}
   1.197 +firethorn.user.data=${userdata:?}
   1.198 +firethorn.user.user=${useruser:?}
   1.199 +firethorn.user.pass=${userpass:?}
   1.200 +
   1.201 +firethorn.tapschema.resource.name=${tapschemajdbc}
   1.202 +firethorn.tapschema.database.name=${tapschemadata}
   1.203 +firethorn.tapschema.database.host=${tapschemahost}
   1.204 +firethorn.tapschema.database.port=${tapschemaport}
   1.205 +firethorn.tapschema.database.user=${tapschemauser}
   1.206 +firethorn.tapschema.database.pass=${tapschemapass}
   1.207 +
   1.208 +firethorn.admin.community=${admingroup:?}
   1.209 +firethorn.admin.user.name=${adminuser:?}
   1.210 +firethorn.admin.user.pass=${adminpass:?}
   1.211 +
   1.212 +firethorn.guest.community=${guestgroup:?}
   1.213 +
   1.214 +adql.query.delay.first=${adql_query_delay_first:?}
   1.215 +
   1.216 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.217 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.218 +	
   1.219 +
   1.220 +EOF
   1.221 +
   1.222 +
   1.223 +
   1.224 +## ------------------------------------------------------------------------------------------
   1.225 +## Create our compose env files.
   1.226 +## ------------------------------------------------------------------------------------------
   1.227 +
   1.228 +source "${HOME:?}/chain.properties"
   1.229 +cat > "${HOME:?}/.env" << EOF
   1.230 +
   1.231 +buildtag=${branch:?}
   1.232 +endpoint=${endpoint:?}
   1.233 +
   1.234 +
   1.235 +metadata=postgres
   1.236 +metauser=${metauser:?}
   1.237 +metapass=${metapass:?}
   1.238 +firethorn.meta.type=pgsql
   1.239 +firethorn.meta.driver=org.postgresql.Driver
   1.240 +firethorn.meta.host=bethany
   1.241 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   1.242 +firethorn.meta.data=postgres
   1.243 +firethorn.meta.user=${metauser:?}
   1.244 +firethorn.meta.pass=${metapass:?}
   1.245 +
   1.246 +
   1.247 +datadata=${datadata:?}
   1.248 +datauser=${datauser:?}
   1.249 +datapass=${datapass:?}
   1.250 +datahost=${datahost:?}
   1.251 +datacatalog=${datacatalog:?}
   1.252 +datatype=mssql
   1.253 +
   1.254 +
   1.255 +tapschemadata=${tapschemadata:?}
   1.256 +tapschemauser=${tapschemauser:?}
   1.257 +tapschemapass=${tapschemapass:?}
   1.258 +tapschemahost=${tapschemahost}
   1.259 +tapschemaport=${tapschemaport:?}
   1.260 +tapschemajdbc=${tapschemajdbc:?}
   1.261 +tapschematype=${tapschematype:?}
   1.262 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   1.263 +firethorn.tapschema.database.name=${tapschemadata:?}
   1.264 +firethorn.tapschema.database.host=${tapschemahost:?}
   1.265 +firethorn.tapschema.database.port=5432
   1.266 +firethorn.tapschema.database.user=${tapschemauser:?}
   1.267 +firethorn.tapschema.database.pass=${tapschemapass:?}
   1.268 +firethorn.tapschema.database.type=pgsql
   1.269 +
   1.270 +userhost=${userhost:?}
   1.271 +userdata=${userdata:?}
   1.272 +useruser=${useruser:?}
   1.273 +userpass=${userpass:?}
   1.274 +usertype=mssql
   1.275 +firethorn.user.type=mssql
   1.276 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.277 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.278 +firethorn.user.host=${userhost:?}
   1.279 +firethorn.user.data=${userdata:?}
   1.280 +firethorn.user.user=${useruser:?}
   1.281 +firethorn.user.pass=${userpass:?}
   1.282 +
   1.283 +
   1.284 +admingroup=${admingroup:?}
   1.285 +adminuser=${adminuser:?}
   1.286 +adminpass=${adminpass:?}
   1.287 +guestgroup=${guestgroup:?}
   1.288 +firethorn.admin.community=${admingroup:?}
   1.289 +firethorn.admin.user.name=${adminuser:?}
   1.290 +firethorn.admin.user.pass=${adminpass:?}
   1.291 +firethorn.guest.community=${guestgroup:?}
   1.292 +
   1.293 +
   1.294 +adql_query_delay_first=${adql_query_delay_first:?}
   1.295 +adql.query.delay.first=${adql_query_delay_first:?}
   1.296 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   1.297 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   1.298 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.299 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.300 +
   1.301 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   1.302 +
   1.303 +firethornhost=gillian
   1.304 +
   1.305 +EOF
   1.306 +
   1.307 +
   1.308 +source "${HOME:?}/chain.properties"
   1.309 +cat > "${HOME:?}/.tapschema_env" << EOF
   1.310 +
   1.311 +POSTGRES_DB=${tapschemadatabase}
   1.312 +POSTGRES_USER=${tapschemauser}
   1.313 +POSTGRES_PASSWORD=${tapschemapass}
   1.314 +
   1.315 +EOF
   1.316 +
   1.317 +
   1.318 +source "${HOME:?}/chain.properties"
   1.319 +cat > "${HOME:?}/.metadata_env" << EOF
   1.320 +
   1.321 +POSTGRES_DB=${metadata}
   1.322 +POSTGRES_USER=${metauser}
   1.323 +POSTGRES_PASSWORD=${metapass}
   1.324 +         
   1.325 +EOF
   1.326 +
   1.327 +
   1.328 +
   1.329 +
   1.330 +## -------------------------------------------------------------------------------------------
   1.331 +## Fetch the TAP Deployer
   1.332 +## -------------------------------------------------------------------------------------------
   1.333 +
   1.334 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   1.335 +
   1.336 +
   1.337 +
   1.338 +
   1.339 +
   1.340 +
   1.341 +## -------------------------------------------------------------------------------------------
   1.342 +## Deploy Swarm stack
   1.343 +## -------------------------------------------------------------------------------------------
   1.344 +
   1.345 +docker stack deploy -c docker-compose.yml ft
   1.346 +
   1.347 +	# Creating network ft_miriam
   1.348 +	# Creating network ft_rosana
   1.349 +	# Creating service ft_jarmila
   1.350 +	# Creating service ft_gillian
   1.351 +	# Creating service ft_firethorn-py
   1.352 +	# Creating service ft_bethany
   1.353 +	# Creating service ft_carolina
   1.354 +
   1.355 +
   1.356 +
   1.357 +## -------------------------------------------------------------------------------------------
   1.358 +## To remove all docker services 
   1.359 +## -------------------------------------------------------------------------------------------
   1.360 +
   1.361 +## docker service rm $(docker service ls -q)
   1.362 +
   1.363 +
   1.364 +
   1.365 +## -------------------------------------------------------------------------------------------
   1.366 +## To list all docker services 
   1.367 +## -------------------------------------------------------------------------------------------
   1.368 +
   1.369 +## docker service ls
   1.370 +
   1.371 +
   1.372 +
   1.373 +docker ps
   1.374 +
   1.375 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                            PORTS               NAMES
   1.376 +4db9c13b1f0d        firethorn/firethorn:2.1.34      "/bin/sh -c '/var/lo…"   5 seconds ago       Up 3 seconds (health: starting)   8080/tcp            ft_gillian.1.su28yys3as7foaqdb0b9nqlwx
   1.377 +fe80420092a1        firethorn/ogsadai:2.1.34        "/bin/sh -c '/var/lo…"   8 seconds ago       Up 6 seconds (health: starting)   8080/tcp            ft_jarmila.1.0dbei14did74h1iu7ku93gx7a
   1.378 +a2c23f565cb8        firethorn/postgres:2.1.34       "docker-entrypoint.s…"   10 seconds ago      Up 8 seconds                      5432/tcp            ft_carolina.1.wx5wtw407thlr128yvh5t9lhm
   1.379 +a85761628bbb        firethorn/postgres:2.1.34       "docker-entrypoint.s…"   12 seconds ago      Up 8 seconds                      5432/tcp            ft_bethany.1.xd20qmguubfyopfy9b7sin4sk
   1.380 +5adc907ffcee        firethorn/firethorn-py:2.1.34   "python3"                15 seconds ago      Up 13 seconds                                         ft_firethorn-py.1.gc6fl7q80yr231lr0kd0pesql
   1.381 +
   1.382 +
   1.383 +
   1.384 +## ------------------------------------------------------------------------
   1.385 +## Create TAP service with firethorn-utils
   1.386 +## ------------------------------------------------------------------------
   1.387 +
   1.388 +docker exec -it ft_firethorn-py.1.gc6fl7q80yr231lr0kd0pesql bash
   1.389 +
   1.390 +
   1.391 +root@firethorn-py:/home# python3
   1.392 +
   1.393 +
   1.394 +import firethorn
   1.395 +import firethorn_utils.configurator as configur
   1.396 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   1.397 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   1.398 +configurator =  configur.Configurator(ft)
   1.399 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/osa-tap.json")
   1.400 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/vsa-tap.json")
   1.401 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/wsa-tap.json")
   1.402 +configurator.load_resources("https://raw.githubusercontent.com/wfau/metadata/master/firethorn/config/ssa-tap.json")
   1.403 +
   1.404 +
   1.405 +
   1.406 +
   1.407 +## OSA
   1.408 +
   1.409 +Importing JDBC Schema: ATLASDR1
   1.410 +Importing JDBC Schema: ATLASDR2
   1.411 +Importing JDBC Schema: ATLASDR3
   1.412 +Importing JDBC Schema: ATLASDR4
   1.413 +Importing JDBC Schema: 2MASS
   1.414 +Importing JDBC Schema: 2XMM
   1.415 +Importing JDBC Schema: WISE
   1.416 +Importing JDBC Schema: VIKINGDR3
   1.417 +Importing JDBC Schema: VHSDR1
   1.418 +Importing JDBC Schema: MGC
   1.419 +Importing JDBC Schema: GLIMPSE
   1.420 +Importing JDBC Schema: FIRST
   1.421 +Importing JDBC Schema: DENIS
   1.422 +Importing JDBC Schema: VPHASDR3
   1.423 +Importing JDBC Schema: VVVDR4
   1.424 +ERROR:root:HTTP Error 404: 
   1.425 +Traceback (most recent call last):
   1.426 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   1.427 +    with urllib.request.urlopen( req, data) as response:
   1.428 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   1.429 +    return opener.open(url, data, timeout)
   1.430 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   1.431 +    response = meth(req, response)
   1.432 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   1.433 +    'http', request, response, code, msg, hdrs)
   1.434 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   1.435 +    return self._call_chain(*args)
   1.436 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   1.437 +    result = func(*args)
   1.438 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   1.439 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   1.440 +urllib.error.HTTPError: HTTP Error 404: 
   1.441 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   1.442 +Traceback (most recent call last):
   1.443 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   1.444 +    response_json_list = json.loads(response.text)
   1.445 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   1.446 +    return _default_decoder.decode(s)
   1.447 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   1.448 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   1.449 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   1.450 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   1.451 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   1.452 +Importing JDBC Schema: SDSSDR7
   1.453 +Importing JDBC Schema: SDSSDR9
   1.454 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   1.455 +
   1.456 +
   1.457 +
   1.458 +## VSA
   1.459 +
   1.460 +Importing JDBC Schema: VVVDR1
   1.461 +Importing JDBC Schema: VVVDR2
   1.462 +Importing JDBC Schema: VVVDR4
   1.463 +ERROR:root:HTTP Error 404: 
   1.464 +Traceback (most recent call last):
   1.465 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   1.466 +    with urllib.request.urlopen( req, data) as response:
   1.467 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   1.468 +    return opener.open(url, data, timeout)
   1.469 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   1.470 +    response = meth(req, response)
   1.471 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   1.472 +    'http', request, response, code, msg, hdrs)
   1.473 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   1.474 +    return self._call_chain(*args)
   1.475 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   1.476 +    result = func(*args)
   1.477 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   1.478 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   1.479 +urllib.error.HTTPError: HTTP Error 404: 
   1.480 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   1.481 +Traceback (most recent call last):
   1.482 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   1.483 +    response_json_list = json.loads(response.text)
   1.484 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   1.485 +    return _default_decoder.decode(s)
   1.486 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   1.487 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   1.488 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   1.489 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   1.490 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   1.491 +Importing JDBC Schema: VHSDR1
   1.492 +Importing JDBC Schema: VHSDR2
   1.493 +Importing JDBC Schema: VHSDR3
   1.494 +Importing JDBC Schema: VMCDR1
   1.495 +Importing JDBC Schema: VMCDR2
   1.496 +Importing JDBC Schema: VMCDR3
   1.497 +Importing JDBC Schema: VMCDR4
   1.498 +Importing JDBC Schema: VIKINGDR2
   1.499 +Importing JDBC Schema: VIKINGDR3
   1.500 +Importing JDBC Schema: VIKINGDR4
   1.501 +Importing JDBC Schema: VIDEODR2
   1.502 +Importing JDBC Schema: VIDEODR3
   1.503 +Importing JDBC Schema: VIDEODR4
   1.504 +Importing JDBC Schema: VIDEODR5
   1.505 +Importing JDBC Schema: UKIDSSDR4PLUS
   1.506 +Importing JDBC Schema: UKIDSSDR5PLUS
   1.507 +Importing JDBC Schema: UKIDSSDR6PLUS
   1.508 +Importing JDBC Schema: UKIDSSDR7PLUS
   1.509 +Importing JDBC Schema: UKIDSSDR8PLUS
   1.510 +Importing JDBC Schema: UKIDSSDR9PLUS
   1.511 +Importing JDBC Schema: UKIDSSDR10PLUS
   1.512 +Importing JDBC Schema: SDSSDR1
   1.513 +Importing JDBC Schema: SDSSDR3
   1.514 +Importing JDBC Schema: SDSSDR5
   1.515 +Importing JDBC Schema: SDSSDR7
   1.516 +Importing JDBC Schema: SDSSDR8
   1.517 +Importing JDBC Schema: SDSSDR9
   1.518 +Importing JDBC Schema: 2MASS
   1.519 +Importing JDBC Schema: ROSAT
   1.520 +Importing JDBC Schema: FIRST
   1.521 +Importing JDBC Schema: IRAS
   1.522 +Importing JDBC Schema: SSA
   1.523 +Importing JDBC Schema: GLIMPSE
   1.524 +Importing JDBC Schema: 2XMM
   1.525 +Importing JDBC Schema: MGC
   1.526 +Importing JDBC Schema: Stripe82
   1.527 +Importing JDBC Schema: WISE
   1.528 +TAP Service available at: http://gillian:8080/firethorn/tap/57/
   1.529 +
   1.530 +
   1.531 +
   1.532 +
   1.533 +## WSA 
   1.534 +
   1.535 +Importing JDBC Schema: UKIDSSDR1PLUS
   1.536 +Importing JDBC Schema: UKIDSSDR2PLUS
   1.537 +Importing JDBC Schema: UKIDSSDR3PLUS
   1.538 +Importing JDBC Schema: UKIDSSDR4PLUS
   1.539 +Importing JDBC Schema: UKIDSSDR5PLUS
   1.540 +Importing JDBC Schema: UKIDSSDR6PLUS
   1.541 +Importing JDBC Schema: UKIDSSDR7PLUS
   1.542 +Importing JDBC Schema: UKIDSSDR8PLUS
   1.543 +Importing JDBC Schema: UKIDSSDR9PLUS
   1.544 +Importing JDBC Schema: UKIDSSDR10PLUS
   1.545 +Importing JDBC Schema: SDSSDR1
   1.546 +Importing JDBC Schema: SDSSDR3
   1.547 +Importing JDBC Schema: SDSSDR5
   1.548 +Importing JDBC Schema: SDSSDR7
   1.549 +Importing JDBC Schema: SDSSDR8
   1.550 +Importing JDBC Schema: SDSSDR9
   1.551 +Importing JDBC Schema: 2MASS
   1.552 +Importing JDBC Schema: ROSAT
   1.553 +Importing JDBC Schema: FIRST
   1.554 +Importing JDBC Schema: IRAS
   1.555 +Importing JDBC Schema: SSA
   1.556 +Importing JDBC Schema: GLIMPSE
   1.557 +Importing JDBC Schema: 2XMM
   1.558 +Importing JDBC Schema: MGC
   1.559 +Importing JDBC Schema: Stripe82
   1.560 +Importing JDBC Schema: WISE
   1.561 +Importing JDBC Schema: WFCAMCAL08B
   1.562 +TAP Service available at: http://gillian:8080/firethorn/tap/60/
   1.563 +
   1.564 +
   1.565 +## SSA
   1.566 +
   1.567 +Importing JDBC Schema: SSA
   1.568 +Importing JDBC Schema: 2MASS
   1.569 +Importing JDBC Schema: 2MPZ
   1.570 +Importing JDBC Schema: WISExSCOSPZ
   1.571 +Importing JDBC Schema: SDSSDR1
   1.572 +Importing JDBC Schema: SDSSDR5
   1.573 +Importing JDBC Schema: SDSSDR7
   1.574 +Importing JDBC Schema: SDSSDR8
   1.575 +Importing JDBC Schema: SDSSDR9
   1.576 +Importing JDBC Schema: 6dF
   1.577 +TAP Service available at: http://gillian:8080/firethorn/tap/63/
   1.578 +
   1.579 +
   1.580 +
   1.581 +
   1.582 +
   1.583 +
   1.584 +# -----------------------------------------------------
   1.585 +# WFAU TAP Service endpoints
   1.586 +# -----------------------------------------------------
   1.587 +
   1.588 +TAP Service available at: http://gillian:8080/firethorn/tap/54/ - OSA
   1.589 +TAP Service available at: http://gillian:8080/firethorn/tap/57/ - VSA
   1.590 +TAP Service available at: http://gillian:8080/firethorn/tap/60/ - WSA
   1.591 +TAP Service available at: http://gillian:8080/firethorn/tap/63/ - SSA
   1.592 +
   1.593 +
   1.594 +
   1.595 +# -----------------------------------------------------
   1.596 +# Update the Proxy Entry
   1.597 +# -----------------------------------------------------
   1.598 +ssh Stevedore@Acilamwen 
   1.599 +
   1.600 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   1.601 +
   1.602 +..
   1.603 +    ## PROXY ENTRIES HERE ##
   1.604 +    ProxyPassMatch    ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   1.605 +    ProxyPassReverse  ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   1.606 +
   1.607 +    ProxyPassMatch    ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   1.608 +    ProxyPassReverse  ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   1.609 +
   1.610 +    ProxyPassMatch    ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   1.611 +    ProxyPassReverse  ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   1.612 +
   1.613 +    ProxyPassMatch    ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   1.614 +    ProxyPassReverse  ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   1.615 +
   1.616 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   1.617 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   1.618 +
   1.619 +
   1.620 +...   
   1.621 +
   1.622 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   1.623 +[Stevedore@Acilamwen ~]$ docker exec -it  apache bash
   1.624 +[root@bc8d6979e451 /]# nano /etc/httpd/conf.d/tap.roe.ac.uk.conf
   1.625 +
   1.626 +..
   1.627 +
   1.628 +    ## PROXY ENTRIES HERE ##
   1.629 +    ProxyPassMatch    ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   1.630 +    ProxyPassReverse  ^/osa\/(.*)$  http://lothigometh:8080/firethorn/tap/54/$1
   1.631 +
   1.632 +    ProxyPassMatch    ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   1.633 +    ProxyPassReverse  ^/vsa\/(.*)$  http://lothigometh:8080/firethorn/tap/57/$1
   1.634 +
   1.635 +    ProxyPassMatch    ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   1.636 +    ProxyPassReverse  ^/wsa\/(.*)$  http://lothigometh:8080/firethorn/tap/60/$1
   1.637 +
   1.638 +    ProxyPassMatch    ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   1.639 +    ProxyPassReverse  ^/ssa\/(.*)$  http://lothigometh:8080/firethorn/tap/63/$1
   1.640 +
   1.641 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   1.642 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://lothigometh:8080/firethorn/$1
   1.643 +
   1.644 +
   1.645 +
   1.646 +..
   1.647 +[root@bc8d6979e451 /]# exit
   1.648 +exit
   1.649 +
   1.650 +[Stevedore@Acilamwen ~]$ docker kill --signal 'USR1' 'apache'
   1.651 +
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/stv/20190720-Test-Firethorn-Unresponsive.txt	Sun Jul 21 01:29:40 2019 +0300
     2.3 @@ -0,0 +1,118 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#  
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#
    2.25 +
    2.26 +############################################################################
    2.27 +#####     Test to cause Firethorn Service to become unresponsive       #####
    2.28 +############################################################################
    2.29 +
    2.30 +
    2.31 +
    2.32 +## Test tap.roe.ac.uk on local machine
    2.33 +
    2.34 +## ------------------------------------------------------------------------
    2.35 +## Run Container on local machine , test Firethorn on a public IP
    2.36 +## ------------------------------------------------------------------------
    2.37 +
    2.38 +docker run -it firethorn/firethorn-py:2.1.34 python3
    2.39 +
    2.40 +
    2.41 +
    2.42 +import os                                                                       
    2.43 +from multiprocessing import Pool                                                
    2.44 +
    2.45 +process_list = range(100)
    2.46 +	
    2.47 +def run_process(process_counter):                                                             
    2.48 +    print ("Starting RUN # " + str(process_counter))
    2.49 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=54 -u= -p= -g= -v=True -m=sync')
    2.50 +    print ("Ended RUN # " + str(process_counter))
    2.51 +
    2.52 +
    2.53 +pool = Pool(processes=20)                                                        
    2.54 +pool.map(run_process, process_list)   
    2.55 +
    2.56 +
    2.57 +## Leave running for up to a few hours
    2.58 +
    2.59 +
    2.60 +
    2.61 +
    2.62 +
    2.63 +
    2.64 +
    2.65 +## Test development deployment
    2.66 +
    2.67 +## ------------------------------------------------------------------------
    2.68 +## Run Container on local machine , test Firethorn on a public IP
    2.69 +## ------------------------------------------------------------------------
    2.70 +
    2.71 +Create a Firethorn chain as described:
    2.72 +http://wfau.metagrid.co.uk/code/firethorn/file/9fd44371154b/doc/notes/stv/20190718-Firethorn-Swarm.txt
    2.73 +
    2.74 +
    2.75 +## Fetch the container name of firethorn.py
    2.76 +
    2.77 +
    2.78 +docker ps
    2.79 +CONTAINER ID        IMAGE                                     COMMAND                  CREATED             STATUS                  PORTS               NAMES
    2.80 +4eb9467315fd        firethorn/firethorn:latest                "/bin/sh -c '/var/lo…"   21 hours ago        Up 21 hours (healthy)   8080/tcp            ft_gillian.1.mqv7eoh2gdb0ydy7ektlzc6iu
    2.81 +c95dc31a37d7        firethorn/ogsadai:2.1.33-stv-tap-swarm    "/bin/sh -c '/var/lo…"   21 hours ago        Up 21 hours (healthy)   8080/tcp            ft_jarmila.1.v8rbyw4x3xkfnwpluu9kwi8nf
    2.82 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm   "docker-entrypoint.s…"   46 hours ago        Up 46 hours             5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
    2.83 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm   "docker-entrypoint.s…"   46 hours ago        Up 46 hours             5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
    2.84 +59b0ce098117        firethorn/firethorn-py:latest             "python3"                46 hours ago        Up 46 hours                                 ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
    2.85 +
    2.86 +
    2.87 +
    2.88 +
    2.89 +## Bash into Firethorn.py container
    2.90 +
    2.91 +docker exec -it ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj bash
    2.92 +
    2.93 +
    2.94 +
    2.95 +
    2.96 +root@firethorn-py:/home# python3
    2.97 +Python 3.5.2 (default, Nov 12 2018, 13:43:14) 
    2.98 +[GCC 5.4.0 20160609] on linux
    2.99 +Type "help", "copyright", "credits" or "license" for more information.
   2.100 +>>> 
   2.101 +
   2.102 +
   2.103 +
   2.104 +import os                                                                       
   2.105 +from multiprocessing import Pool                                                
   2.106 +
   2.107 +process_list = range(100)
   2.108 +	
   2.109 +def run_process(process_counter):                                                             
   2.110 +    print ("Starting RUN # " + str(process_counter))
   2.111 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://gillian:8080/firethorn -r=54 -u=chae1aixuh5exae5Udoh -p=EiB0shoegahfeWahx1ai -g=oowe1eedaeFei1tohtai -v=True -m=sync')
   2.112 +    print ("Ended RUN # " + str(process_counter))
   2.113 +
   2.114 +
   2.115 +pool = Pool(processes=20)                                                        
   2.116 +pool.map(run_process, process_list)  
   2.117 +
   2.118 +
   2.119 +
   2.120 +## Leave running until all queries start failing
   2.121 +