firethorn

changeset 4263:550c349452fd

Merged upstream changes
author Dave Morris <dmr@roe.ac.uk>
date Tue Jul 02 04:12:49 2019 +0200 (20 months ago)
parents ba11bf1b3fdf 52e38613deef
children e99f391f91b4
files
line diff
     1.1 --- a/.hgtags	Tue Jul 02 04:12:16 2019 +0200
     1.2 +++ b/.hgtags	Tue Jul 02 04:12:49 2019 +0200
     1.3 @@ -44,3 +44,5 @@
     1.4  40233dc8a520ce2b24812b862786b12fccf303b7 version-2.1.29
     1.5  e85c79f225d2307b217ab304ed9e20f8c9a1e85b version-2.1.30
     1.6  da9e0640f31b0f40f440642ad5ce4ebdbc356711 version-2.1.31
     1.7 +919418055ef1ff5e3e10e6ad91509251b9991000 version-2.1.32
     1.8 +e9d24ed64319a22c950e80e5f8444f760056cb22 version-2.1.33
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/stv/20190530-Block-IP.txt	Tue Jul 02 04:12:49 2019 +0200
     2.3 @@ -0,0 +1,43 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2014, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#  
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#
    2.25 +
    2.26 +
    2.27 +## Block Validator from Japan on the WFAU TAP services proxy 
    2.28 +
    2.29 +ssh Stevedore@Acilamwen
    2.30 +
    2.31 +   docker exec -it apache bash
    2.32 +
    2.33 +
    2.34 +   nano /etc/httpd/conf.d/tap.roe.ac.uk.conf
    2.35 +   ...
    2.36 +   ## Add line
    2.37 +
    2.38 +	    <Proxy *>
    2.39 +		    order allow,deny
    2.40 +		    Deny from 133.40.215.109
    2.41 +		    Allow from all
    2.42 +	    </Proxy>
    2.43 +   ...
    2.44 +
    2.45 +exit
    2.46 +docker kill --signal 'USR1' 'apache
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/doc/notes/stv/20190613-Testing-2.txt	Tue Jul 02 04:12:49 2019 +0200
     3.3 @@ -0,0 +1,674 @@
     3.4 +#
     3.5 +# <meta:header>
     3.6 +#   <meta:licence>
     3.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     3.8 +#
     3.9 +#     This information is free software: you can redistribute it and/or modify
    3.10 +#     it under the terms of the GNU General Public License as published by
    3.11 +#     the Free Software Foundation, either version 3 of the License, or
    3.12 +#     (at your option) any later version.
    3.13 +#
    3.14 +#     This information is distributed in the hope that it will be useful,
    3.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    3.17 +#     GNU General Public License for more details.
    3.18 +#
    3.19 +#     You should have received a copy of the GNU General Public License
    3.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    3.21 +#   </meta:licence>
    3.22 +# </meta:header>
    3.23 +#
    3.24 +#
    3.25 +
    3.26 +
    3.27 +
    3.28 +## Test: Run  30 TAP Validators in Parallel
    3.29 +## VM: Arraybwyn
    3.30 +## tap.roe.ac.uk
    3.31 +## Firethorn Version: 2.1.33 
    3.32 +
    3.33 +## Causes the Firethorn Container to stop responding, but not consistently
    3.34 +
    3.35 +python3
    3.36 +
    3.37 +import os                                                                       
    3.38 +from multiprocessing import Pool                                                
    3.39 +
    3.40 +process_list = range(100)
    3.41 +	
    3.42 +def run_process(process_counter):                                                             
    3.43 +    print ("Starting RUN # " + str(process_counter))
    3.44 +    os.system('python3 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=60 -u= -p= -g= -m=sync')                                       
    3.45 +    print ("Ended RUN # " + str(process_counter))
    3.46 +
    3.47 +
    3.48 +pool = Pool(processes=30)                                                        
    3.49 +pool.map(run_process, process_list)   
    3.50 +
    3.51 +
    3.52 +## Test 1
    3.53 +## ----------------------------------------------------------------------------------------------------------------------------------------------------
    3.54 +## 22:09 - Started Test
    3.55 +## Run 30 parallel TAP Validators, running sync queries on all of the tables of the "VSA" resource on tap.roe.ac.uk
    3.56 +
    3.57 +## For the first few queries, we get a timeout error in python
    3.58 +## After that there are a number of successfull queries for a few minutes, after which the failed queries (timeouts) start again
    3.59 +## Again, after a few minutes we see the same symptoms, a number of successful queries, after which the logs show a list of failed queries
    3.60 +
    3.61 +
    3.62 +## 22:43
    3.63 +## http://tap.roe.ac.uk/osa/tables  and http://tap.roe.ac.uk/firethorn/system/info return successfully
    3.64 +
    3.65 +## 00:16
    3.66 +## http://tap.roe.ac.uk/osa/tables  and http://tap.roe.ac.uk/firethorn/system/info return are both very slow to respond, but eventually do repond successfully
    3.67 +## All queries are now failing
    3.68 +## http://tap.roe.ac.uk/firethorn/system/info  is responding, but slow
    3.69 +
    3.70 +
    3.71 +## Run was stopped during the night (machine shutdown)
    3.72 +## Services survived
    3.73 +
    3.74 +## Sample of the output is shown below:
    3.75 +
    3.76 +Starting RUN # 0
    3.77 +Starting RUN # 1
    3.78 +Starting RUN # 2
    3.79 +Starting RUN # 3
    3.80 +Starting RUN # 4
    3.81 +Starting RUN # 5
    3.82 +Starting RUN # 6
    3.83 +Starting RUN # 7
    3.84 +Starting RUN # 8
    3.85 +Starting RUN # 9
    3.86 +Starting RUN # 10
    3.87 +Starting RUN # 11
    3.88 +Starting RUN # 12
    3.89 +Starting RUN # 13
    3.90 +Starting RUN # 14
    3.91 +Starting RUN # 15
    3.92 +Starting RUN # 16
    3.93 +Starting RUN # 17
    3.94 +Starting RUN # 18
    3.95 +Starting RUN # 19
    3.96 +Starting RUN # 20
    3.97 +Starting RUN # 21
    3.98 +Starting RUN # 22
    3.99 +Starting RUN # 23
   3.100 +Starting RUN # 24
   3.101 +Starting RUN # 25
   3.102 +Starting RUN # 26
   3.103 +Starting RUN # 27
   3.104 +Starting RUN # 28
   3.105 +Starting RUN # 29
   3.106 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.107 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.108 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.109 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.110 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.111 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.112 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.113 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.114 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.115 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.116 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.117 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.118 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.119 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.120 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.121 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.122 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.123 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.124 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.125 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.126 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.127 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.128 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.129 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.130 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.131 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.132 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.133 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.134 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.135 +--- Starting validation on Resource: http://tap.roe.ac.uk/firethorn/adql/resource/60---
   3.136 +Testing [BestDR1.SpecObjAll]
   3.137 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.138 +Testing [BestDR1.SpecObjAll]
   3.139 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.140 +Testing [BestDR1.SpecObjAll]
   3.141 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.142 +Testing [BestDR1.SpecObjAll]
   3.143 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.144 +Testing [BestDR1.SpecObjAll]
   3.145 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.146 +Testing [BestDR1.SpecObjAll]
   3.147 +Testing [BestDR1.SpecObjAll]
   3.148 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.149 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.150 +Testing [BestDR1.SpecObjAll]
   3.151 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.152 +Testing [BestDR1.SpecObjAll]
   3.153 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.154 +Testing [BestDR1.SpecObjAll]
   3.155 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.156 +Testing [BestDR1.SpecObjAll]
   3.157 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.158 +Testing [BestDR1.SpecObjAll]
   3.159 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.160 +Testing [BestDR1.SpecObjAll]
   3.161 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.162 +Testing [BestDR1.SpecObjAll]
   3.163 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.164 +Testing [BestDR1.SpecObjAll]
   3.165 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.166 +Testing [BestDR1.SpecObjAll]
   3.167 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.168 +Testing [BestDR1.SpecObjAll]
   3.169 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.170 +Testing [BestDR1.SpecObjAll]
   3.171 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.172 +Testing [BestDR1.SpecObjAll]
   3.173 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.174 +Testing [BestDR1.SpecObjAll]
   3.175 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.176 +Testing [BestDR1.SpecObjAll]
   3.177 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.178 +Testing [BestDR1.SpecObjAll]
   3.179 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.180 +Testing [BestDR1.SpecObjAll]
   3.181 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.182 +Testing [BestDR1.SpecObjAll]
   3.183 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.184 +Testing [BestDR1.SpecObjAll]
   3.185 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.186 +Testing [BestDR1.SpecObjAll]
   3.187 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.188 +Testing [BestDR1.SpecObjAll]
   3.189 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.190 +Testing [BestDR1.SpecObjAll]
   3.191 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.192 +Testing [BestDR1.SpecObjAll]
   3.193 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.194 +Testing [BestDR1.SpecObjAll]
   3.195 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecObjAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.196 +ERROR:root:<urlopen error timed out>
   3.197 +Traceback (most recent call last):
   3.198 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.199 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.200 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.201 +    return opener.open(url, data, timeout)
   3.202 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.203 +    response = self._open(req, data)
   3.204 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.205 +    '_open', req)
   3.206 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.207 +    result = func(*args)
   3.208 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.209 +    return self.do_open(http.client.HTTPConnection, req)
   3.210 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.211 +    r = h.getresponse()
   3.212 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.213 +    response.begin()
   3.214 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.215 +    version, status, reason = self._read_status()
   3.216 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.217 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.218 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.219 +    return self._sock.recv_into(b)
   3.220 +socket.timeout: timed out
   3.221 +
   3.222 +During handling of the above exception, another exception occurred:
   3.223 +
   3.224 +Traceback (most recent call last):
   3.225 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   3.226 +    self.votable = Table.read(full_url, format="votable")
   3.227 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   3.228 +    out = registry.read(cls, *args, **kwargs)
   3.229 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   3.230 +    data = reader(*args, **kwargs)
   3.231 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   3.232 +    input = parse(input, table_id=table_id)
   3.233 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   3.234 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   3.235 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.236 +    return next(self.gen)
   3.237 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   3.238 +    with _convert_to_fd_or_read_function(source) as fd:
   3.239 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.240 +    return next(self.gen)
   3.241 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   3.242 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   3.243 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.244 +    return next(self.gen)
   3.245 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   3.246 +    timeout=remote_timeout)
   3.247 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1097, in download_file
   3.248 +    raise urllib.error.URLError(e)
   3.249 +urllib.error.URLError: <urlopen error timed out>
   3.250 +Rowcount:-1
   3.251 +<class 'Exception'>
   3.252 +{'BestDR1.SpecObjAll': '<urlopen error timed out>'}
   3.253 +Table query completed after 10.047759532928467 seconds
   3.254 +
   3.255 +
   3.256 +
   3.257 +......................................................................................................................................................
   3.258 +
   3.259 +
   3.260 +
   3.261 +
   3.262 +
   3.263 +Testing [BestDR1.Field]
   3.264 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.265 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.MatchHead&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.266 +|===============================================================================================================================================================================================| 2.7k/2.7k (100.00%)         0s
   3.267 +Rowcount:5
   3.268 +Table query completed after 7.05977725982666 seconds
   3.269 +Testing [BestDR1.History]
   3.270 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.271 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.272 +|===============================================================================================================================================================================================| 2.2k/2.2k (100.00%)         0s
   3.273 +Rowcount:0
   3.274 +Table query completed after 7.143743515014648 seconds
   3.275 +Testing [BestDR1.Field]
   3.276 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.277 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.278 +|===============================================================================================================================================================================================| 2.2k/2.2k (100.00%)         0s
   3.279 +Rowcount:0
   3.280 +Table query completed after 7.266925096511841 seconds
   3.281 +Testing [BestDR1.Field]
   3.282 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.283 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.284 +|===============================================================================================================================================================================================| 2.2k/2.2k (100.00%)         0s
   3.285 +Rowcount:0
   3.286 +Table query completed after 8.249078750610352 seconds
   3.287 +Testing [BestDR1.Field]
   3.288 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.289 +ERROR:root:<urlopen error timed out>
   3.290 +Traceback (most recent call last):
   3.291 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.292 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.293 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.294 +    return opener.open(url, data, timeout)
   3.295 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.296 +    response = self._open(req, data)
   3.297 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.298 +    '_open', req)
   3.299 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.300 +    result = func(*args)
   3.301 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.302 +    return self.do_open(http.client.HTTPConnection, req)
   3.303 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.304 +    r = h.getresponse()
   3.305 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.306 +    response.begin()
   3.307 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.308 +    version, status, reason = self._read_status()
   3.309 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.310 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.311 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.312 +    return self._sock.recv_into(b)
   3.313 +socket.timeout: timed out
   3.314 +
   3.315 +During handling of the above exception, another exception occurred:
   3.316 +
   3.317 +Traceback (most recent call last):
   3.318 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   3.319 +    self.votable = Table.read(full_url, format="votable")
   3.320 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   3.321 +    out = registry.read(cls, *args, **kwargs)
   3.322 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   3.323 +    data = reader(*args, **kwargs)
   3.324 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   3.325 +    input = parse(input, table_id=table_id)
   3.326 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   3.327 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   3.328 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.329 +    return next(self.gen)
   3.330 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   3.331 +    with _convert_to_fd_or_read_function(source) as fd:
   3.332 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.333 +    return next(self.gen)
   3.334 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   3.335 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   3.336 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.337 +    return next(self.gen)
   3.338 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   3.339 +    timeout=remote_timeout)
   3.340 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1097, in download_file
   3.341 +    raise urllib.error.URLError(e)
   3.342 +urllib.error.URLError: <urlopen error timed out>
   3.343 +Rowcount:-1
   3.344 +<class 'Exception'>
   3.345 +{'BestDR1.SpecObjAll': '<urlopen error timed out>', 'BestDR1.Field': '<urlopen error timed out>'}
   3.346 +Table query completed after 10.040127038955688 seconds
   3.347 +Testing [BestDR1.SpecLineAll]
   3.348 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.SpecLineAll&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.349 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.350 +|===============================================================================================================================================================================================| 2.2k/2.2k (100.00%)         0s
   3.351 +Rowcount:0
   3.352 +Table query completed after 11.048564672470093 seconds
   3.353 +Testing [BestDR1.Field]
   3.354 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.355 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.History&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.356 +|===============================================================================================================================================================================================| 2.2k/2.2k (100.00%)         0s
   3.357 +Rowcount:0
   3.358 +Table query completed after 11.176760911941528 seconds
   3.359 +Testing [BestDR1.Field]
   3.360 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR1.Field&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.361 +ERROR:root:<urlopen error timed out>
   3.362 +Traceback (most recent call last):
   3.363 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.364 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.365 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.366 +    return opener.open(url, data, timeout)
   3.367 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.368 +    response = self._open(req, data)
   3.369 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.370 +    '_open', req)
   3.371 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.372 +    result = func(*args)
   3.373 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.374 +    return self.do_open(http.client.HTTPConnection, req)
   3.375 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.376 +    r = h.getresponse()
   3.377 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.378 +    response.begin()
   3.379 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.380 +    version, status, reason = self._read_status()
   3.381 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.382 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.383 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.384 +    return self._sock.recv_into(b)
   3.385 +socket.timeout: timed out
   3.386 +
   3.387 +During handling of the above exception, another exception occurred:
   3.388 +
   3.389 +Traceback (most recent call last):
   3.390 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   3.391 +    self.votable = Table.read(full_url, format="votable")
   3.392 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   3.393 +    out = registry.read(cls, *args, **kwargs)
   3.394 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   3.395 +    data = reader(*args, **kwargs)
   3.396 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   3.397 +    input = parse(input, table_id=table_id)
   3.398 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   3.399 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   3.400 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.401 +    return next(self.gen)
   3.402 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   3.403 +    with _convert_to_fd_or_read_function(source) as fd:
   3.404 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.405 +    return next(self.gen)
   3.406 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   3.407 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   3.408 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.409 +    return next(self.gen)
   3.410 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   3.411 +    timeout=remote_timeout)
   3.412 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1097, in download_file
   3.413 +    raise urllib.error.URLError(e)
   3.414 +urllib.error.URLError: <urlopen error timed out>
   3.415 +ERROR:root:<urlopen error timed out>
   3.416 +Traceback (most recent call last):
   3.417 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.418 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.419 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.420 +    return opener.open(url, data, timeout)
   3.421 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.422 +    response = self._open(req, data)
   3.423 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.424 +    '_open', req)
   3.425 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.426 +    result = func(*args)
   3.427 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.428 +    return self.do_open(http.client.HTTPConnection, req)
   3.429 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.430 +    r = h.getresponse()
   3.431 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.432 +    response.begin()
   3.433 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.434 +    version, status, reason = self._read_status()
   3.435 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.436 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.437 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.438 +    return self._sock.recv_into(b)
   3.439 +socket.timeout: timed out
   3.440 +
   3.441 +During handling of the above exception, another exception occurred:
   3.442 +
   3.443 +Traceback (most recent call last):
   3.444 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   3.445 +    self.votable = Table.read(full_url, format="votable")
   3.446 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   3.447 +    out = registry.read(cls, *args, **kwargs)
   3.448 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   3.449 +    data = reader(*args, **kwargs)
   3.450 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   3.451 +    input = parse(input, table_id=table_id)
   3.452 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   3.453 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   3.454 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.455 +    return next(self.gen)
   3.456 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   3.457 +    with _convert_to_fd_or_read_function(source) as fd:
   3.458 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.459 +    return next(self.gen)
   3.460 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   3.461 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   3.462 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.463 +    return next(self.gen)
   3.464 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   3.465 +    timeout=remote_timeout)
   3.466 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1097, in download_file
   3.467 +    raise urllib.error.URLError(e)
   3.468 +urllib.error.URLError: <urlopen error timed out>Rowcount:-1
   3.469 +
   3.470 +<class 'Exception'>
   3.471 +{'BestDR1.SpecObjAll': '<urlopen error timed out>', 'BestDR1.Field': '<urlopen error timed out>'}
   3.472 +Table query completed after 10.040641069412231 seconds
   3.473 +
   3.474 +
   3.475 +..............................................................................................................................................................................................................
   3.476 +
   3.477 +Testing [BestDR3.PhotoFamily]
   3.478 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.479 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE -Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingBoundary&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.480 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.481 +Rowcount:5
   3.482 +Table query completed after 9.394372701644897 seconds
   3.483 +Testing [BestDR3.TilingMask]
   3.484 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.485 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE [Done]
   3.486 +Rowcount:5
   3.487 +Table query completed after 9.834657669067383 seconds
   3.488 +Testing [BestDR3.PhotoFamily]
   3.489 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.490 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE [Done]
   3.491 +Rowcount:5
   3.492 +Table query completed after 9.936257362365723 seconds
   3.493 +Testing [BestDR3.PhotoFamily]
   3.494 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.495 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.496 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.497 +Rowcount:5
   3.498 +Table query completed after 8.035722017288208 seconds
   3.499 +Testing [BestDR3.TiledTarget]
   3.500 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.501 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE -Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.502 +|===============================================================================================================================================================================================| 4.2k/4.2k (100.00%)         0s
   3.503 +Rowcount:5
   3.504 +Table query completed after 8.153093099594116 seconds
   3.505 +Testing [BestDR3.SpecObj]
   3.506 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.507 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE [Done]
   3.508 +Rowcount:5
   3.509 +Table query completed after 10.541740655899048 seconds
   3.510 +Testing [BestDR3.PhotoFamily]
   3.511 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.512 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.513 +|===============================================================================================================================================================================================| 4.2k/4.2k (100.00%)         0s
   3.514 +Rowcount:5
   3.515 +Table query completed after 7.451206207275391 seconds
   3.516 +Testing [BestDR3.SpecObj]
   3.517 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.518 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingBoundary&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.519 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.520 +Rowcount:5
   3.521 +Table query completed after 7.12007737159729 seconds
   3.522 +Testing [BestDR3.TilingMask]
   3.523 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.524 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE -Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingBoundary&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.525 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.526 +Rowcount:5
   3.527 +Table query completed after 7.456395626068115 seconds
   3.528 +Testing [BestDR3.TilingMask]
   3.529 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.530 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE [Done]
   3.531 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE [Done]
   3.532 +Rowcount:5
   3.533 +Table query completed after 9.467787027359009 seconds
   3.534 +Testing [BestDR3.PhotoFamily]
   3.535 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.536 +Rowcount:5
   3.537 +Table query completed after 8.390213251113892 seconds
   3.538 +Testing [BestDR3.PhotoFamily]
   3.539 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.540 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.541 +|===============================================================================================================================================================================================| 4.2k/4.2k (100.00%)         0s
   3.542 +Rowcount:5
   3.543 +Table query completed after 7.233114957809448 seconds
   3.544 +Testing [BestDR3.SpecObj]
   3.545 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.546 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingBoundary&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.547 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.548 +Rowcount:5
   3.549 +Table query completed after 7.394808053970337 seconds
   3.550 +Testing [BestDR3.TilingMask]
   3.551 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.552 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.553 +|===============================================================================================================================================================================================| 4.2k/4.2k (100.00%)         0s
   3.554 +Rowcount:5
   3.555 +Table query completed after 7.578145265579224 seconds
   3.556 +Testing [BestDR3.SpecObj]
   3.557 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.558 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.559 +|===============================================================================================================================================================================================| 4.2k/4.2k (100.00%)         0s
   3.560 +Rowcount:5
   3.561 +Table query completed after 7.709306478500366 seconds
   3.562 +Testing [BestDR3.SpecObj]
   3.563 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.SpecObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.564 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.565 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.566 +Rowcount:5
   3.567 +Table query completed after 7.802347660064697 seconds
   3.568 +Testing [BestDR3.TiledTarget]
   3.569 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.570 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.571 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.572 +Rowcount:5
   3.573 +Table query completed after 8.726522445678711 seconds
   3.574 +Testing [BestDR3.TiledTarget]
   3.575 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.576 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.577 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.578 +Rowcount:5
   3.579 +Table query completed after 8.938718318939209 seconds
   3.580 +Testing [BestDR3.TiledTarget]
   3.581 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.582 +Downloading http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TilingMask&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.583 +|===============================================================================================================================================================================================| 4.7k/4.7k (100.00%)         0s
   3.584 +Rowcount:5
   3.585 +Table query completed after 9.069983959197998 seconds
   3.586 +Testing [BestDR3.TiledTarget]
   3.587 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.TiledTarget&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.588 +ERROR:root:<urlopen error timed out>
   3.589 +Traceback (most recent call last):
   3.590 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.591 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.592 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.593 +    return opener.open(url, data, timeout)
   3.594 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.595 +    response = self._open(req, data)
   3.596 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.597 +    '_open', req)
   3.598 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.599 +    result = func(*args)
   3.600 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.601 +    return self.do_open(http.client.HTTPConnection, req)
   3.602 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.603 +    r = h.getresponse()
   3.604 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.605 +    response.begin()
   3.606 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.607 +    version, status, reason = self._read_status()
   3.608 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.609 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.610 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.611 +    return self._sock.recv_into(b)
   3.612 +socket.timeout: timed out
   3.613 +
   3.614 +During handling of the above exception, another exception occurred:
   3.615 +
   3.616 +Traceback (most recent call last):
   3.617 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   3.618 +    self.votable = Table.read(full_url, format="votable")
   3.619 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   3.620 +    out = registry.read(cls, *args, **kwargs)
   3.621 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   3.622 +    data = reader(*args, **kwargs)
   3.623 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   3.624 +    input = parse(input, table_id=table_id)
   3.625 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   3.626 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   3.627 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.628 +    return next(self.gen)
   3.629 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   3.630 +    with _convert_to_fd_or_read_function(source) as fd:
   3.631 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.632 +    return next(self.gen)
   3.633 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   3.634 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   3.635 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   3.636 +    return next(self.gen)
   3.637 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   3.638 +    timeout=remote_timeout)
   3.639 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1097, in download_file
   3.640 +    raise urllib.error.URLError(e)
   3.641 +urllib.error.URLError: <urlopen error timed out>
   3.642 +Rowcount:-1
   3.643 +<class 'Exception'>
   3.644 +{'BestDR1.SpecObjAll': '<urlopen error timed out>', 'BestDR1.Field': '<urlopen error timed out>', 'BestDR1.SpecLineAll': '<urlopen error timed out>', 'BestDR1.DataConstants': '<urlopen error timed out>', 'BestDR1.TilingNote': '<urlopen error timed out>', 'BestDR1.SiteDiagnostics': '<urlopen error timed out>', 'BestDR1.PhotoObjAll': '<urlopen error timed out>', 'BestDR1.Frame': '<urlopen error timed out>', 'BestDR1."Zone"': '<urlopen error timed out>', 'BestDR1.PartitionMap': '<urlopen error timed out>', 'BestDR1.DBObjects': '<urlopen error timed out>', 'BestDR1.Sector': '<urlopen error timed out>', 'BestDR1.DBObjectDescription': '<urlopen error timed out>', 'BestDR1.Glossary': '<urlopen error timed out>', 'BestDR1.TilingRun': '<urlopen error timed out>', 'BestDR1.Chunk': '<urlopen error timed out>', 'BestDR1.Segment': '<urlopen error timed out>', 'BestDR1."First"': '<urlopen error timed out>', 'BestDR1.PlateX': '<urlopen error timed out>', 'BestDR3.Field': '<urlopen error timed out>', 'BestDR3.FieldProfile': '<urlopen error timed out>', 'BestDR3.FileGroupMap': '<urlopen error timed out>', 'BestDR3."First"': '<urlopen error timed out>', 'BestDR3.Frame': '<urlopen error timed out>', 'BestDR3.Glossary': '<urlopen error timed out>', 'BestDR3.History': '<urlopen error timed out>', 'BestDR3.HoleObj': '<urlopen error timed out>', 'BestDR3.PhotoObjAll': '<urlopen error timed out>', 'BestDR3.PhotoProfile': '<urlopen error timed out>', 'BestDR3.PhotoTag': '<urlopen error timed out>', 'BestDR3.Photoz': '<urlopen error timed out>', 'BestDR3.PlateX': '<urlopen error timed out>', 'BestDR3.ProfileDefs': '<urlopen error timed out>', 'BestDR3.PubHistory': '<urlopen error timed out>', 'BestDR3.QsoCatalogAll': '<urlopen error timed out>', 'BestDR3.QsoConcordance': '<urlopen error timed out>', 'BestDR3.QueryResults': '<urlopen error timed out>', 'BestDR3.RC3': '<urlopen error timed out>', 'BestDR3.PhotoFamily': '<urlopen error timed out>'}
   3.645 +Table query completed after 10.04263687133789 seconds
   3.646 +Testing [BestDR3.PhotoObj]
   3.647 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR3.PhotoObj&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
   3.648 +ERROR:root:<urlopen error timed out>
   3.649 +Traceback (most recent call last):
   3.650 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   3.651 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   3.652 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   3.653 +    return opener.open(url, data, timeout)
   3.654 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   3.655 +    response = self._open(req, data)
   3.656 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   3.657 +    '_open', req)
   3.658 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   3.659 +    result = func(*args)
   3.660 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   3.661 +    return self.do_open(http.client.HTTPConnection, req)
   3.662 +  File "/usr/lib/python3.6/urllib/request.py", line 1321, in do_open
   3.663 +    r = h.getresponse()
   3.664 +  File "/usr/lib/python3.6/http/client.py", line 1331, in getresponse
   3.665 +    response.begin()
   3.666 +  File "/usr/lib/python3.6/http/client.py", line 297, in begin
   3.667 +    version, status, reason = self._read_status()
   3.668 +  File "/usr/lib/python3.6/http/client.py", line 258, in _read_status
   3.669 +    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
   3.670 +  File "/usr/lib/python3.6/socket.py", line 586, in readinto
   3.671 +    return self._sock.recv_into(b)
   3.672 +socket.timeout: timed out
   3.673 +
   3.674 +
   3.675 +
   3.676 +
   3.677 +
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/doc/notes/stv/20190613-Testing.txt	Tue Jul 02 04:12:49 2019 +0200
     4.3 @@ -0,0 +1,54 @@
     4.4 +#
     4.5 +# <meta:header>
     4.6 +#   <meta:licence>
     4.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     4.8 +#
     4.9 +#     This information is free software: you can redistribute it and/or modify
    4.10 +#     it under the terms of the GNU General Public License as published by
    4.11 +#     the Free Software Foundation, either version 3 of the License, or
    4.12 +#     (at your option) any later version.
    4.13 +#
    4.14 +#     This information is distributed in the hope that it will be useful,
    4.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    4.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    4.17 +#     GNU General Public License for more details.
    4.18 +#
    4.19 +#     You should have received a copy of the GNU General Public License
    4.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    4.21 +#   </meta:licence>
    4.22 +# </meta:header>
    4.23 +#
    4.24 +#
    4.25 +
    4.26 +## Run 10 TAP queries in Parallel using stilts.jar
    4.27 +## Try to emulate what the Japanese validator is doing
    4.28 +for run in {1..10}
    4.29 +do
    4.30 +  java -jar stilts.jar tapquery tapurl=http://tap.roe.ac.uk/vsa sync=true adql="SELECT TOP 1 * from UKIDSSDR4PLUS.UKIDSSDetection" &;
    4.31 +done
    4.32 +
    4.33 +
    4.34 +
    4.35 +## --------------------------------------------------------------------
    4.36 +
    4.37 +
    4.38 +## Run  20 TAP Validators in Paralle
    4.39 +## Causes the Firethorn Container to stop responding, but not consistently
    4.40 +
    4.41 +python3
    4.42 +
    4.43 +
    4.44 +import os                                                                       
    4.45 +from multiprocessing import Pool                                                
    4.46 +
    4.47 +process_list = range(100)
    4.48 +	
    4.49 +def run_process(process_counter):                                                             
    4.50 +    print ("Starting RUN # " + str(process_counter))
    4.51 +    os.system('python3 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=60 -u= -p= -g= -m=sync')                                       
    4.52 +    print ("Ended RUN # " + str(process_counter))
    4.53 +
    4.54 +
    4.55 +pool = Pool(processes=20)                                                        
    4.56 +pool.map(run_process, process_list)   
    4.57 +
     5.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     5.2 +++ b/doc/notes/stv/20190619-TAP-Deploy.txt	Tue Jul 02 04:12:49 2019 +0200
     5.3 @@ -0,0 +1,502 @@
     5.4 +#
     5.5 +# <meta:header>
     5.6 +#   <meta:licence>
     5.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     5.8 +#
     5.9 +#     This information is free software: you can redistribute it and/or modify
    5.10 +#     it under the terms of the GNU General Public License as published by
    5.11 +#     the Free Software Foundation, either version 3 of the License, or
    5.12 +#     (at your option) any later version.
    5.13 +#
    5.14 +#     This information is distributed in the hope that it will be useful,
    5.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    5.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    5.17 +#     GNU General Public License for more details.
    5.18 +#  
    5.19 +#     You should have received a copy of the GNU General Public License
    5.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    5.21 +#   </meta:licence>
    5.22 +# </meta:header>
    5.23 +#
    5.24 +#
    5.25 +
    5.26 +##########################################################
    5.27 +##### Notes on Deploying WFAU TAP Services on trop02 #####
    5.28 +##########################################################
    5.29 +
    5.30 +stv@trop02:~$ createvm
    5.31 +
    5.32 +INFO : Base pool  [base]
    5.33 +INFO : Live pool  [live]
    5.34 +INFO : Connection [qemu:///system]
    5.35 +
    5.36 +INFO : Data path  [/var/local/projects/ischnura/github/src/dat]
    5.37 +
    5.38 +INFO : Machines   [/var/local/projects/ischnura/github/src/dat/tropo-machines.txt]
    5.39 +INFO : Template   [/var/local/projects/ischnura/github/src/dat/tropo-template.xml]
    5.40 +
    5.41 +[1] Araybwyn
    5.42 +[2] Lothigometh
    5.43 +[3] Ulov
    5.44 +[4] Dwiema
    5.45 +[5] Ibalehar
    5.46 +[6] araybwyn
    5.47 +[7] Siamond
    5.48 +[8] Acilamwen
    5.49 +Select machine name (1) 1
    5.50 +setlocale: No such file or directory
    5.51 +
    5.52 +[1] fedora-28-docker-base-20180708.qcow
    5.53 +[2] fedora-28-8G-docker-base-20181016.qcow
    5.54 +[3] fedora-28-32G-docker-base-20181016.qcow
    5.55 +[4] fedora-28-16G-docker-base-20181016.qcow
    5.56 +Select base image (1) 3
    5.57 +setlocale: No such file or directory
    5.58 +
    5.59 +INFO : Node name [Araybwyn]
    5.60 +INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    5.61 +INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    5.62 +INFO : Disc name [Araybwyn.qcow]
    5.63 +INFO : Disc size [32GiB]
    5.64 +
    5.65 +INFO : MAC  [52:54:00:02:02:08]
    5.66 +INFO : IPv4 [192.168.202.8]
    5.67 +INFO : IPv6 []
    5.68 +
    5.69 +Create virtual machine (Y/n)Y
    5.70 +
    5.71 +Found existing virtual machine [Araybwyn]
    5.72 +Delete existing virtual machine (Y/n)Y
    5.73 +setlocale: No such file or directory
    5.74 +Stopping virtual machine [Araybwyn]
    5.75 +setlocale: No such file or directory
    5.76 +Domain a65ad4ca-0662-4c43-a32c-91e4180d87bc destroyed
    5.77 +Delete associated storage (Y/n)Y
    5.78 +Deleting virtual machine [Araybwyn]
    5.79 +setlocale: No such file or directory
    5.80 +Domain a65ad4ca-0662-4c43-a32c-91e4180d87bc has been undefined
    5.81 +Volume 'vda'(/libvirt/storage/live/Araybwyn.qcow) removed.
    5.82 +Volume 'vdb'(/var/lib/libvirt/images/init/Araybwyn.iso) removed.
    5.83 +
    5.84 +Creating new volume [Araybwyn.qcow]
    5.85 +setlocale: No such file or directory
    5.86 +Vol Araybwyn.qcow created
    5.87 +setlocale: No such file or directory
    5.88 +Name:           Araybwyn.qcow
    5.89 +Type:           file
    5.90 +Capacity:       32.00 GiB
    5.91 +Allocation:     196.00 KiB
    5.92 +
    5.93 +setlocale: No such file or directory
    5.94 +Vol Araybwyn.iso created
    5.95 +setlocale: No such file or directory
    5.96 +setlocale: No such file or directory
    5.97 +setlocale: No such file or directory
    5.98 +Creating new virtual machine [Araybwyn]
    5.99 +setlocale: No such file or directory
   5.100 +Domain Araybwyn defined from /tmp/tmp.G2hFEjLYp6
   5.101 +Starting new virtual machine [Araybwyn]
   5.102 +setlocale: No such file or directory
   5.103 +Domain Araybwyn started
   5.104 +
   5.105 +
   5.106 +[Stevedore@Araybwyn ~]$ 
   5.107 +
   5.108 +
   5.109 +# -----------------------------------------------------
   5.110 +# Add secret function.
   5.111 +# -----------------------------------------------------
   5.112 +
   5.113 +    secrethost='stv@shepseskaf.roe.ac.uk'
   5.114 +    secretfile='${HOME:?}/secret.store.20181212'
   5.115 +
   5.116 +    secret()
   5.117 +    {
   5.118 +        local key=${1:?}
   5.119 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
   5.120 +    }
   5.121 +
   5.122 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
   5.123 +
   5.124 +
   5.125 +
   5.126 +# -----------------------------------------------------
   5.127 +# Set the target branch and builtag
   5.128 +# -----------------------------------------------------
   5.129 +
   5.130 +
   5.131 +    export buildtag=2.1.33
   5.132 +    export branch=default
   5.133 +
   5.134 +
   5.135 +
   5.136 +
   5.137 +# -----------------------------------------------------
   5.138 +# Create our chain.properties file
   5.139 +# -----------------------------------------------------
   5.140 +
   5.141 +cat > "${HOME:?}/chain.properties" << EOF
   5.142 +
   5.143 +metadata=postgres
   5.144 +metauser=$(pwgen 20 1)
   5.145 +metapass=$(pwgen 20 1)
   5.146 +
   5.147 +userhost=$(secret 'firethorn.user.host')
   5.148 +userdata=$(secret 'firethorn.user.data')
   5.149 +useruser=$(secret 'firethorn.user.user')
   5.150 +userpass=$(secret 'firethorn.user.pass')
   5.151 +usertype=mssql
   5.152 +
   5.153 +datahost=$(secret 'firethorn.data.host')
   5.154 +datadata=$(secret 'firethorn.data.data')
   5.155 +datacatalog=$(secret 'firethorn.data.catalog')
   5.156 +datatype=$(secret 'firethorn.data.type')
   5.157 +datauser=$(secret 'firethorn.data.user')
   5.158 +datapass=$(secret 'firethorn.data.pass')
   5.159 +
   5.160 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   5.161 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   5.162 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   5.163 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   5.164 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   5.165 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   5.166 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   5.167 +
   5.168 +tunneluser=$(secret 'ssh.tunnel.user')
   5.169 +tunnelhost=$(secret 'ssh.tunnel.host')
   5.170 +
   5.171 +admingroup=$(pwgen 20 1)	
   5.172 +adminuser=$(pwgen 20 1)
   5.173 +adminpass=$(pwgen 20 1)
   5.174 +
   5.175 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   5.176 + 
   5.177 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   5.178 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   5.179 +
   5.180 +guestgroup=friends
   5.181 +endpoint=$(secret 'endpoint')
   5.182 +
   5.183 +EOF
   5.184 +
   5.185 +
   5.186 +
   5.187 +# -----------------------------------------------------
   5.188 +# Create our Firethorn properties.
   5.189 +# -----------------------------------------------------
   5.190 +
   5.191 +source "${HOME:?}/chain.properties"
   5.192 +cat > "${HOME:?}/firethorn.properties" << EOF
   5.193 +
   5.194 +firethorn.meta.type=pgsql
   5.195 +firethorn.meta.driver=org.postgresql.Driver
   5.196 +firethorn.meta.host=bethany
   5.197 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   5.198 +firethorn.meta.data=postgres
   5.199 +firethorn.meta.user=${metauser:?}
   5.200 +firethorn.meta.pass=${metapass:?}
   5.201 +
   5.202 +firethorn.user.type=mssql
   5.203 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   5.204 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   5.205 +firethorn.user.host=${userhost:?}
   5.206 +firethorn.user.data=${userdata:?}
   5.207 +firethorn.user.user=${useruser:?}
   5.208 +firethorn.user.pass=${userpass:?}
   5.209 +
   5.210 +firethorn.tapschema.resource.name=${tapschemajdbc}
   5.211 +firethorn.tapschema.database.name=${tapschemadata}
   5.212 +firethorn.tapschema.database.host=${tapschemahost}
   5.213 +firethorn.tapschema.database.port=${tapschemaport}
   5.214 +firethorn.tapschema.database.user=${tapschemauser}
   5.215 +firethorn.tapschema.database.pass=${tapschemapass}
   5.216 +
   5.217 +firethorn.admin.community=${admingroup:?}
   5.218 +firethorn.admin.user.name=${adminuser:?}
   5.219 +firethorn.admin.user.pass=${adminpass:?}
   5.220 +
   5.221 +firethorn.guest.community=${guestgroup:?}
   5.222 +
   5.223 +adql.query.delay.first=${adql_query_delay_first:?}
   5.224 +
   5.225 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   5.226 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   5.227 +	
   5.228 +
   5.229 +EOF
   5.230 +
   5.231 +
   5.232 +
   5.233 +# -----------------------------------------------------
   5.234 +# Create our compose env file.
   5.235 +# -----------------------------------------------------
   5.236 +
   5.237 +source "${HOME:?}/chain.properties"
   5.238 +cat > "${HOME:?}/.env" << EOF
   5.239 +
   5.240 +	buildtag=${branch:?}
   5.241 +
   5.242 +	metadata=postgres
   5.243 +	metauser=${metauser:?}
   5.244 +	metapass=${metapass:?}
   5.245 +
   5.246 +        datadata=${datadata:?}
   5.247 +        datauser=${datauser:?}
   5.248 +        datapass=${datapass:?}
   5.249 +	datahost=${datahost:?}
   5.250 +	datacatalog=${datacatalog:?}
   5.251 +	datatype=mssql
   5.252 +
   5.253 +	tapschemadata=${tapschemadata:?}
   5.254 +	tapschemauser=${tapschemauser:?}
   5.255 +	tapschemapass=${tapschemapass:?}
   5.256 +	tapschemahost=${tapschemahost}
   5.257 +	tapschemaport=${tapschemaport:?}
   5.258 +	tapschemajdbc=${tapschemajdbc:?}
   5.259 +	tapschematype=${tapschematype:?}
   5.260 +
   5.261 +	userhost=${userhost:?}
   5.262 +	userdata=${userdata:?}
   5.263 +	useruser=${useruser:?}
   5.264 +	userpass=${userpass:?}
   5.265 +        usertype=mssql
   5.266 +
   5.267 +	admingroup=${admingroup:?}
   5.268 +	adminuser=${adminuser:?}
   5.269 +	adminpass=${adminpass:?}
   5.270 +	guestgroup=${guestgroup:?}
   5.271 +
   5.272 +	adql_query_delay_first=${adql_query_delay_first:?}
   5.273 +	
   5.274 +	firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   5.275 +	firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   5.276 +
   5.277 +	endpoint=${endpoint:?}
   5.278 +EOF
   5.279 +
   5.280 +
   5.281 +
   5.282 +
   5.283 +# -----------------------------------------------------
   5.284 +# Fetch the TAP Deployer
   5.285 +# -----------------------------------------------------
   5.286 +
   5.287 +
   5.288 + wget http://wfau.metagrid.co.uk/code/firethorn/raw-file/20b8901d807c/docker/compose/deploy/tap-deployer.yml  -O  docker-compose.yml
   5.289 +
   5.290 +
   5.291 +
   5.292 +# -----------------------------------------------------
   5.293 +# Run Docker Compose tap-deployer.yml from firethorn.py
   5.294 +# -----------------------------------------------------
   5.295 +
   5.296 +
   5.297 +source "${HOME:?}/chain.properties"
   5.298 +
   5.299 +docker-compose --file "docker-compose.yml" run firethorn-py
   5.300 +
   5.301 +import firethorn
   5.302 +import firethorn_utils.configurator as configur
   5.303 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   5.304 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   5.305 +configurator =  configur.Configurator(ft)
   5.306 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   5.307 +
   5.308 +Importing JDBC Schema: ATLASDR1
   5.309 +Importing JDBC Schema: ATLASDR2
   5.310 +Importing JDBC Schema: ATLASDR3
   5.311 +Importing JDBC Schema: 2MASS
   5.312 +Importing JDBC Schema: 2XMM
   5.313 +Importing JDBC Schema: WISE
   5.314 +Importing JDBC Schema: VIKINGDR3
   5.315 +Importing JDBC Schema: VHSDR1
   5.316 +Importing JDBC Schema: MGC
   5.317 +Importing JDBC Schema: GLIMPSE
   5.318 +Importing JDBC Schema: FIRST
   5.319 +Importing JDBC Schema: DENIS
   5.320 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   5.321 +
   5.322 +configurator.load_resources("/home/firethorn.py/firethorn/data/ssa-tap.json")
   5.323 +Importing JDBC Schema: SSA
   5.324 +Importing JDBC Schema: 2MASS
   5.325 +Importing JDBC Schema: 2MPZ
   5.326 +Importing JDBC Schema: WISExSCOSPZ
   5.327 +Importing JDBC Schema: SDSSDR1
   5.328 +Importing JDBC Schema: SDSSDR5
   5.329 +Importing JDBC Schema: SDSSDR7
   5.330 +Importing JDBC Schema: SDSSDR8
   5.331 +Importing JDBC Schema: SDSSDR9
   5.332 +Importing JDBC Schema: 6dF
   5.333 +TAP Service available at: http://gillian:8080/firethorn/tap/57/
   5.334 +
   5.335 +configurator.load_resources("/home/firethorn.py/firethorn/data/vsa-tap.json")
   5.336 +Importing JDBC Schema: VVVDR1
   5.337 +Importing JDBC Schema: VVVDR2
   5.338 +Importing JDBC Schema: VVVDR4
   5.339 +ERROR:root:HTTP Error 404: 
   5.340 +Traceback (most recent call last):
   5.341 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 90, in get_json
   5.342 +    with urllib.request.urlopen( req, data) as response:
   5.343 +  File "/usr/lib/python3.5/urllib/request.py", line 163, in urlopen
   5.344 +    return opener.open(url, data, timeout)
   5.345 +  File "/usr/lib/python3.5/urllib/request.py", line 472, in open
   5.346 +    response = meth(req, response)
   5.347 +  File "/usr/lib/python3.5/urllib/request.py", line 582, in http_response
   5.348 +    'http', request, response, code, msg, hdrs)
   5.349 +  File "/usr/lib/python3.5/urllib/request.py", line 510, in error
   5.350 +    return self._call_chain(*args)
   5.351 +  File "/usr/lib/python3.5/urllib/request.py", line 444, in _call_chain
   5.352 +    result = func(*args)
   5.353 +  File "/usr/lib/python3.5/urllib/request.py", line 590, in http_error_default
   5.354 +    raise HTTPError(req.full_url, code, msg, hdrs, fp)
   5.355 +urllib.error.HTTPError: HTTP Error 404: 
   5.356 +ERROR:root:Expecting value: line 1 column 1 (char 0)
   5.357 +Traceback (most recent call last):
   5.358 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_resource.py", line 112, in import_jdbc_schema
   5.359 +    response_json_list = json.loads(response.text)
   5.360 +  File "/usr/lib/python3.5/json/__init__.py", line 319, in loads
   5.361 +    return _default_decoder.decode(s)
   5.362 +  File "/usr/lib/python3.5/json/decoder.py", line 339, in decode
   5.363 +    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
   5.364 +  File "/usr/lib/python3.5/json/decoder.py", line 357, in raw_decode
   5.365 +    raise JSONDecodeError("Expecting value", s, err.value) from None
   5.366 +json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
   5.367 +Importing JDBC Schema: VHSDR1
   5.368 +Importing JDBC Schema: VHSDR2
   5.369 +Importing JDBC Schema: VHSDR3
   5.370 +Importing JDBC Schema: VMCDR1
   5.371 +Importing JDBC Schema: VMCDR2
   5.372 +Importing JDBC Schema: VMCDR3
   5.373 +Importing JDBC Schema: VMCDR4
   5.374 +Importing JDBC Schema: VIKINGDR2
   5.375 +Importing JDBC Schema: VIKINGDR3
   5.376 +Importing JDBC Schema: VIKINGDR4
   5.377 +Importing JDBC Schema: VIDEODR2
   5.378 +Importing JDBC Schema: VIDEODR3
   5.379 +Importing JDBC Schema: VIDEODR4
   5.380 +Importing JDBC Schema: VIDEODR5
   5.381 +Importing JDBC Schema: UKIDSSDR4PLUS
   5.382 +Importing JDBC Schema: UKIDSSDR5PLUS
   5.383 +Importing JDBC Schema: UKIDSSDR6PLUS
   5.384 +Importing JDBC Schema: UKIDSSDR7PLUS
   5.385 +Importing JDBC Schema: UKIDSSDR8PLUS
   5.386 +Importing JDBC Schema: UKIDSSDR9PLUS
   5.387 +Importing JDBC Schema: UKIDSSDR10PLUS
   5.388 +Importing JDBC Schema: SDSSDR1
   5.389 +Importing JDBC Schema: SDSSDR3
   5.390 +Importing JDBC Schema: SDSSDR5
   5.391 +Importing JDBC Schema: SDSSDR7
   5.392 +Importing JDBC Schema: SDSSDR8
   5.393 +Importing JDBC Schema: SDSSDR9
   5.394 +Importing JDBC Schema: 2MASS
   5.395 +Importing JDBC Schema: ROSAT
   5.396 +Importing JDBC Schema: FIRST
   5.397 +Importing JDBC Schema: IRAS
   5.398 +Importing JDBC Schema: SSA
   5.399 +Importing JDBC Schema: GLIMPSE
   5.400 +Importing JDBC Schema: 2XMM
   5.401 +Importing JDBC Schema: MGC
   5.402 +Importing JDBC Schema: Stripe82
   5.403 +Importing JDBC Schema: WISE
   5.404 +TAP Service available at: http://gillian:8080/firethorn/tap/60/
   5.405 +
   5.406 +configurator.load_resources("/home/firethorn.py/firethorn/data/wsa-tap.json")
   5.407 +Importing JDBC Schema: UKIDSSDR1PLUS
   5.408 +Importing JDBC Schema: UKIDSSDR2PLUS
   5.409 +Importing JDBC Schema: UKIDSSDR3PLUS
   5.410 +Importing JDBC Schema: UKIDSSDR4PLUS
   5.411 +Importing JDBC Schema: UKIDSSDR5PLUS
   5.412 +Importing JDBC Schema: UKIDSSDR6PLUS
   5.413 +Importing JDBC Schema: UKIDSSDR7PLUS
   5.414 +Importing JDBC Schema: UKIDSSDR8PLUS
   5.415 +Importing JDBC Schema: UKIDSSDR9PLUS
   5.416 +Importing JDBC Schema: UKIDSSDR10PLUS
   5.417 +Importing JDBC Schema: SDSSDR1
   5.418 +Importing JDBC Schema: SDSSDR3
   5.419 +Importing JDBC Schema: SDSSDR5
   5.420 +Importing JDBC Schema: SDSSDR7
   5.421 +Importing JDBC Schema: SDSSDR8
   5.422 +Importing JDBC Schema: SDSSDR9
   5.423 +Importing JDBC Schema: 2MASS
   5.424 +Importing JDBC Schema: ROSAT
   5.425 +Importing JDBC Schema: FIRST
   5.426 +Importing JDBC Schema: IRAS
   5.427 +Importing JDBC Schema: SSA
   5.428 +Importing JDBC Schema: GLIMPSE
   5.429 +Importing JDBC Schema: 2XMM
   5.430 +Importing JDBC Schema: MGC
   5.431 +Importing JDBC Schema: Stripe82
   5.432 +Importing JDBC Schema: WISE
   5.433 +TAP Service available at: http://gillian:8080/firethorn/tap/63/
   5.434 +
   5.435 +
   5.436 +
   5.437 +# -----------------------------------------------------
   5.438 +# WFAU TAP Service endpoints
   5.439 +# -----------------------------------------------------
   5.440 +
   5.441 +TAP Service available at: http://gillian:8080/firethorn/tap/54/ - OSA
   5.442 +TAP Service available at: http://gillian:8080/firethorn/tap/57/ - SSA
   5.443 +TAP Service available at: http://gillian:8080/firethorn/tap/60/ - VSA
   5.444 +TAP Service available at: http://gillian:8080/firethorn/tap/63/ - WSA
   5.445 +
   5.446 +
   5.447 +
   5.448 +# -----------------------------------------------------
   5.449 +# Update the Proxy Entry
   5.450 +# -----------------------------------------------------
   5.451 +ssh Stevedore@Acilamwen 
   5.452 +
   5.453 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   5.454 +
   5.455 +..
   5.456 +    ## PROXY ENTRIES HERE ##
   5.457 +    ProxyPassMatch    ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   5.458 +    ProxyPassReverse  ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   5.459 +
   5.460 +    ProxyPassMatch    ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   5.461 +    ProxyPassReverse  ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   5.462 +
   5.463 +    ProxyPassMatch    ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   5.464 +    ProxyPassReverse  ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   5.465 +
   5.466 +    ProxyPassMatch    ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   5.467 +    ProxyPassReverse  ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   5.468 +
   5.469 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   5.470 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   5.471 +
   5.472 +
   5.473 +
   5.474 +...   
   5.475 +
   5.476 +[Stevedore@Acilamwen ~]$ nano tap.roe.ac.uk.conf 
   5.477 +[Stevedore@Acilamwen ~]$ docker exec -it  apache bash
   5.478 +[root@bc8d6979e451 /]# nano /etc/httpd/conf.d/tap.roe.ac.uk.conf
   5.479 +
   5.480 +..
   5.481 +
   5.482 +    ## PROXY ENTRIES HERE ##
   5.483 +    ProxyPassMatch    ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   5.484 +    ProxyPassReverse  ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   5.485 +
   5.486 +    ProxyPassMatch    ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   5.487 +    ProxyPassReverse  ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   5.488 +
   5.489 +    ProxyPassMatch    ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   5.490 +    ProxyPassReverse  ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   5.491 +
   5.492 +    ProxyPassMatch    ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   5.493 +    ProxyPassReverse  ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   5.494 +
   5.495 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   5.496 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   5.497 +
   5.498 +
   5.499 +
   5.500 +..
   5.501 +[root@bc8d6979e451 /]# exit
   5.502 +exit
   5.503 +
   5.504 +[Stevedore@Acilamwen ~]$ docker kill --signal 'USR1' 'apache'
   5.505 +
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/doc/notes/stv/20190625-Testing-TAP-parallel.txt	Tue Jul 02 04:12:49 2019 +0200
     6.3 @@ -0,0 +1,232 @@
     6.4 +#
     6.5 +# <meta:header>
     6.6 +#   <meta:licence>
     6.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     6.8 +#
     6.9 +#     This information is free software: you can redistribute it and/or modify
    6.10 +#     it under the terms of the GNU General Public License as published by
    6.11 +#     the Free Software Foundation, either version 3 of the License, or
    6.12 +#     (at your option) any later version.
    6.13 +#
    6.14 +#     This information is distributed in the hope that it will be useful,
    6.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    6.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    6.17 +#     GNU General Public License for more details.
    6.18 +#
    6.19 +#     You should have received a copy of the GNU General Public License
    6.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    6.21 +#   </meta:licence>
    6.22 +# </meta:header>
    6.23 +#
    6.24 +#
    6.25 +
    6.26 +
    6.27 +
    6.28 +## Test: Run  30 TAP Validators in Parallel
    6.29 +## VM: Arraybwyn
    6.30 +## tap.roe.ac.uk
    6.31 +## Firethorn Version: 2.1.33 
    6.32 +
    6.33 +## Causes the Firethorn Container to stop responding, but not consistently
    6.34 +
    6.35 +python3
    6.36 +
    6.37 +import os                                                                       
    6.38 +from multiprocessing import Pool                                                
    6.39 +
    6.40 +process_list = range(100)
    6.41 +	
    6.42 +def run_process(process_counter):                                                             
    6.43 +    print ("Starting RUN # " + str(process_counter))
    6.44 +    os.system('python3 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=60 -u= -p= -g= -m=sync')                                       
    6.45 +    print ("Ended RUN # " + str(process_counter))
    6.46 +
    6.47 +
    6.48 +pool = Pool(processes=30)                                                        
    6.49 +pool.map(run_process, process_list)   
    6.50 +
    6.51 +..
    6.52 +
    6.53 +## Test started around: 2019-06-25 10:55:00,000:
    6.54 +## Lost connection to VM at 2019-06-25 11:36:28,361
    6.55 +
    6.56 +## Terminal that was following the firethorn-error log failed with: packet_write_wait: Connection to UNKNOWN port 65535: Broken pipe
    6.57 +
    6.58 +2019-06-25 11:36:28,181 ERROR [FireThornTaskExecutor-85] [OgsaBaseResourceEntity] Ping test failed [27564578][DataRequestExecutionResource][404] 
    6.59 +2019-06-25 11:36:28,361 ERROR [FireThornTaskExecutor-53] [OgsaBaseResourceEntity] Ping test failed [27564579][DataRequestExecutionResource][404] 
    6.60 +packet_write_wait: Connection to UNKNOWN port 65535: Broken pipe
    6.61 +
    6.62 +
    6.63 +## stelios@stelios-pc:~$ ssh Stevedore@Araybwyn 
    6.64 +## Does not respond
    6.65 +## curl http://tap.roe.ac.uk/firethorn/system/info does not respond either
    6.66 +
    6.67 +
    6.68 +## At the time where ssh connection was lost, all table queries where failing:
    6.69 +
    6.70 +Testing [BestDR8.PhotoFamily]
    6.71 +Testing [BestDR8.Dependency]
    6.72 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR8.Dependency&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
    6.73 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR8.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
    6.74 +ERROR:root:<urlopen error timed out>
    6.75 +Traceback (most recent call last):
    6.76 +  File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
    6.77 +    encode_chunked=req.has_header('Transfer-encoding'))
    6.78 +  File "/usr/lib/python3.6/http/client.py", line 1239, in request
    6.79 +    self._send_request(method, url, body, headers, encode_chunked)
    6.80 +  File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
    6.81 +    self.endheaders(body, encode_chunked=encode_chunked)
    6.82 +  File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
    6.83 +    self._send_output(message_body, encode_chunked=encode_chunked)
    6.84 +  File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
    6.85 +    self.send(msg)
    6.86 +  File "/usr/lib/python3.6/http/client.py", line 964, in send
    6.87 +    self.connect()
    6.88 +  File "/usr/lib/python3.6/http/client.py", line 936, in connect
    6.89 +    (self.host,self.port), self.timeout, self.source_address)
    6.90 +  File "/usr/lib/python3.6/socket.py", line 724, in create_connection
    6.91 +    raise err
    6.92 +  File "/usr/lib/python3.6/socket.py", line 713, in create_connection
    6.93 +    sock.connect(sa)
    6.94 +socket.timeout: timed out
    6.95 +
    6.96 +During handling of the above exception, another exception occurred:
    6.97 +
    6.98 +Traceback (most recent call last):
    6.99 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   6.100 +    self.votable = Table.read(full_url, format="votable")
   6.101 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   6.102 +    out = registry.read(cls, *args, **kwargs)
   6.103 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   6.104 +    data = reader(*args, **kwargs)
   6.105 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   6.106 +    input = parse(input, table_id=table_id)
   6.107 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   6.108 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   6.109 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   6.110 +    return next(self.gen)
   6.111 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   6.112 +    with _convert_to_fd_or_read_function(source) as fd:
   6.113 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   6.114 +    return next(self.gen)
   6.115 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   6.116 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   6.117 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   6.118 +    return next(self.gen)
   6.119 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   6.120 +    timeout=remote_timeout)
   6.121 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1091, in download_file
   6.122 +    raise e
   6.123 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   6.124 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   6.125 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   6.126 +    return opener.open(url, data, timeout)
   6.127 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   6.128 +    response = self._open(req, data)
   6.129 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   6.130 +    '_open', req)
   6.131 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   6.132 +    result = func(*args)
   6.133 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   6.134 +    return self.do_open(http.client.HTTPConnection, req)
   6.135 +  File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
   6.136 +    raise URLError(err)
   6.137 +urllib.error.URLError: <urlopen error timed out>
   6.138 +Rowcount:-1
   6.139 +<class 'Exception'>
   6.140 +{'BestDR1.Field': '<urlopen error timed out>', 'BestDR1.SpecLineAll': '<urlopen error timed out>', 'BestDR1.DataConstants': '<urlopen error timed out>', 'BestDR1.TilingNote': '<urlopen error timed out>', 'BestDR1.SiteDiagnostics': '<urlopen error timed out>', 'BestDR1.PhotoObjAll': '<urlopen error timed out>', 'BestDR1.Frame': '<urlopen error timed out>', 'BestDR1."Zone"': '<urlopen error timed out>', 'BestDR1.PartitionMap': '<urlopen error timed out>', 'BestDR1.DBObjects': '<urlopen error timed out>', 'BestDR1.Sector': '<urlopen error timed out>', 'BestDR1.DBObjectDescription': '<urlopen error timed out>', 'BestDR1.Glossary': '<urlopen error timed out>', 'BestDR1.TilingRun': '<urlopen error timed out>', 'BestDR1.Chunk': '<urlopen error timed out>', 'BestDR1.Segment': '<urlopen error timed out>', 'BestDR1."First"': '<urlopen error timed out>', 'BestDR1.PlateX': '<urlopen error timed out>', 'BestDR1.PubHistory': '<urlopen error timed out>', 'BestDR1.FieldProfile': '<urlopen error timed out>', 'BestDR1.Best2Sector': '<urlopen error timed out>', 'BestDR1.DBViewCols': '<urlopen error timed out>', 'BestDR1.Mask': '<urlopen error timed out>', 'BestDR1.Target': '<urlopen error timed out>', 'BestDR1.IndexMap': '<urlopen error timed out>', 'BestDR1.TiledTargetAll': '<urlopen error timed out>', 'BestDR1.TargetInfo': '<urlopen error timed out>', 'BestDR1.Sector2Tile': '<urlopen error timed out>', 'BestDR1.SpecPhotoAll': '<urlopen error timed out>', 'BestDR3.Algorithm': '<urlopen error timed out>', 'BestDR3.Field': '<urlopen error timed out>', 'BestDR3.FieldProfile': '<urlopen error timed out>', 'BestDR3.FileGroupMap': '<urlopen error timed out>', 'BestDR3."First"': '<urlopen error timed out>', 'BestDR3.Frame': '<urlopen error timed out>', 'BestDR3.Glossary': '<urlopen error timed out>', 'BestDR3.History': '<urlopen error timed out>', 'BestDR3.HoleObj': '<urlopen error timed out>', 'BestDR3.IndexMap': '<urlopen error timed out>', 'BestDR3.LoadHistory': '<urlopen error timed out>', 'BestDR3.Mask': '<urlopen error timed out>', 'BestDR3.MaskedObject': '<urlopen error timed out>', 'BestDR3."Match"': '<urlopen error timed out>', 'BestDR3.MatchHead': '<urlopen error timed out>', 'BestDR3.PhotoObjAll': '<urlopen error timed out>', 'BestDR3.PhotoProfile': '<urlopen error timed out>', 'BestDR3.PhotoTag': '<urlopen error timed out>', 'BestDR3.Photoz': '<urlopen error timed out>', 'BestDR3.PlateX': '<urlopen error timed out>', 'BestDR3.ProfileDefs': '<urlopen error timed out>', 'BestDR3.PubHistory': '<urlopen error timed out>', 'BestDR3.QsoCatalogAll': '<urlopen error timed out>', 'BestDR3.QsoConcordance': '<urlopen error timed out>', 'BestDR3.QueryResults': '<urlopen error timed out>', 'BestDR3.RC3': '<urlopen error timed out>', 'BestDR3.RecentQueries': '<urlopen error timed out>', 'BestDR3."Region"': '<urlopen error timed out>', 'BestDR3.Rmatrix': '<urlopen error timed out>', 'BestDR3.Rosat': '<urlopen error timed out>', 'BestDR3.RunQA': '<urlopen error timed out>', 'BestDR3.RunShift': '<urlopen error timed out>', 'BestDR3.SDSSConstants': '<urlopen error timed out>', 'BestDR3.Sector': '<urlopen error timed out>', 'BestDR3.SpecObj': '<urlopen error timed out>', 'BestDR3.PhotoFamily': '<urlopen error timed out>', 'BestDR3.PhotoObj': '<urlopen error timed out>', 'BestDR3.PhotoPrimary': '<urlopen error timed out>', 'BestDR3.PhotoSecondary': '<urlopen error timed out>', 'BestDR3.SpecLine': '<urlopen error timed out>', 'BestDR3.SpecPhoto': '<urlopen error timed out>', 'BestDR3.Galaxy': '<urlopen error timed out>', 'BestDR3.Sky': '<urlopen error timed out>', 'BestDR3.Star': '<urlopen error timed out>', 'BestDR3.Unknown': '<urlopen error timed out>', 'BestDR5.SpecObjAll': '<urlopen error timed out>', 'BestDR5.TilingInfo': '<urlopen error timed out>', 'BestDR5.QsoBunch': '<urlopen error timed out>', 'BestDR5.HoleType': '<urlopen error timed out>', 'BestDR5.PhotoStatus': '<urlopen error timed out>', 'BestDR5.SDSSConstants': '<urlopen error timed out>', 'BestDR5.SiteDiagnostics': '<urlopen error timed out>', 'BestDR5.PubHistory': '<urlopen error timed out>', 'BestDR5.MaskType': '<urlopen error timed out>', 'BestDR5.PhotoPrimary': '<urlopen error timed out>', 'BestDR5.History': '<urlopen error timed out>', 'BestDR5.SpecLineIndex': '<urlopen error timed out>', 'BestDR5.Sector': '<urlopen error timed out>', 'BestDR5.PhotoFlags': '<urlopen error timed out>', 'BestDR5.FieldQuality': '<urlopen error timed out>', 'BestDR5.SpecObj': '<urlopen error timed out>', 'BestDR5.PhotoSecondary': '<urlopen error timed out>', 'BestDR5.TilingRun': '<urlopen error timed out>', 'BestDR5.PhotoAux': '<urlopen error timed out>', 'BestDR5.RunShift': '<urlopen error timed out>', 'BestDR5.FieldProfile': '<urlopen error timed out>', 'BestDR5.Frame': '<urlopen error timed out>', 'BestDR5.Region2Box': '<urlopen error timed out>', 'BestDR5.Sky': '<urlopen error timed out>', 'BestDR5.TilingBoundary': '<urlopen error timed out>', 'BestDR5.FramesStatus': '<urlopen error timed out>', 'BestDR5.StripeDefs': '<urlopen error timed out>', 'BestDR5.TableDesc': '<urlopen error timed out>', 'BestDR5.QsoTarget': '<urlopen error timed out>', 'BestDR5.FieldMask': '<urlopen error timed out>', 'BestDR5.TargetParam': '<urlopen error timed out>', 'BestDR5.QsoCatalog': '<urlopen error timed out>', 'BestDR5.Photoz': '<urlopen error timed out>', 'BestDR5."Match"': '<urlopen error timed out>', 'BestDR5.SiteConstants': '<urlopen error timed out>', 'BestDR5.PhotoMode': '<urlopen error timed out>', 'BestDR5."Region"': '<urlopen error timed out>', 'BestDR5.Chunk': '<urlopen error timed out>', 'BestDR5.SpecLine': '<urlopen error timed out>', 'BestDR5.CoordType': '<urlopen error timed out>', 'BestDR5.SpecLineAll': '<urlopen error timed out>', 'BestDR5.StarTag': '<urlopen error timed out>', 'BestDR5.PspStatus': '<urlopen error timed out>', 'BestDR5.Versions': '<urlopen error timed out>', 'BestDR5.BestTarget2Sector': '<urlopen error timed out>', 'BestDR5.Inventory': '<urlopen error timed out>', 'BestDR5.Rmatrix': '<urlopen error timed out>', 'BestDR5.TilingGeometry': '<urlopen error timed out>', 'BestDR5.Segment': '<urlopen error timed out>', 'BestDR5.LoadHistory': '<urlopen error timed out>', 'BestDR5.ProperMotions': '<urlopen error timed out>', 'BestDR5.IndexMap': '<urlopen error timed out>', 'BestDR5.Mask': '<urlopen error timed out>', 'BestDR5.Target': '<urlopen error timed out>', 'BestDR5.SpecPhotoAll': '<urlopen error timed out>', 'BestDR5.DBObjects': '<urlopen error timed out>', 'BestDR5.Neighbors': '<urlopen error timed out>', 'BestDR5.DBColumns': '<urlopen error timed out>', 'BestDR5.ObjMask': '<urlopen error timed out>', 'BestDR5.TilingMask': '<urlopen error timed out>', 'BestDR5.RecentQueries': '<urlopen error timed out>', 'BestDR5.PrimTarget': '<urlopen error timed out>', 'BestDR5.Galaxy': '<urlopen error timed out>', 'BestDR5.MaskedObject': '<urlopen error timed out>', 'BestDR5.MatchHead': '<urlopen error timed out>', 'BestDR5.PhotoProfile': '<urlopen error timed out>', 'BestDR5.USNO': '<urlopen error timed out>', 'BestDR5.TileAll': '<urlopen error timed out>', 'BestDR5.DBViewCols': '<urlopen error timed out>', 'BestDR5.PartitionMap': '<urlopen error timed out>', 'BestDR5.FileGroupMap': '<urlopen error timed out>', 'BestDR5.Glossary': '<urlopen error timed out>', 'BestDR5."Diagnostics"': '<urlopen error timed out>', 'BestDR5.Field': '<urlopen error timed out>', 'BestDR5.TargetInfo': '<urlopen error timed out>', 'BestDR5.DataConstants': '<urlopen error timed out>', 'BestDR5.TilingNote': '<urlopen error timed out>', 'BestDR5.RegionArcs': '<urlopen error timed out>', 'BestDR5.Stetson': '<urlopen error timed out>', 'BestDR5.GalaxyTag': '<urlopen error timed out>', 'BestDR5.Run': '<urlopen error timed out>', 'BestDR5.PhotoAuxAll': '<urlopen error timed out>', 'BestDR5."First"': '<urlopen error timed out>', 'BestDR5.TargRunQA': '<urlopen error timed out>', 'BestDR5.HoleObj': '<urlopen error timed out>', 'BestDR5.QuasarCatalog': '<urlopen error timed out>', 'BestDR5.SpecZStatus': '<urlopen error timed out>', 'BestDR5.PhotoTag': '<urlopen error timed out>', 'BestDR5.Columns': '<urlopen error timed out>', 'BestDR5.SpecLineNames': '<urlopen error timed out>', 'BestDR5.SiteDBs': '<urlopen error timed out>', 'BestDR5.TiledTarget': '<urlopen error timed out>', 'BestDR5.Star': '<urlopen error timed out>', 'BestDR5.QueryResults': '<urlopen error timed out>', 'BestDR5.SpecZWarning': '<urlopen error timed out>', 'BestDR5.QsoCatalogAll': '<urlopen error timed out>', 'BestDR5.RegionConvex': '<urlopen error timed out>', 'BestDR5.Algorithm': '<urlopen error timed out>', 'BestDR5.Photoz2': '<urlopen error timed out>', 'BestDR5."Zone"': '<urlopen error timed out>', 'BestDR5.ImageMask': '<urlopen error timed out>', 'BestDR5.QsoConcordanceAll': '<urlopen error timed out>', 'BestDR5.ProgramType': '<urlopen error timed out>', 'BestDR5.SpecClass': '<urlopen error timed out>', 'BestDR5.PhotoObjAll': '<urlopen error timed out>', 'BestDR5.RC3': '<urlopen error timed out>', 'BestDR5.PhotoType': '<urlopen error timed out>', 'BestDR5.Tile': '<urlopen error timed out>', 'BestDR5.HalfSpace': '<urlopen error timed out>', 'BestDR5.PlateX': '<urlopen error timed out>', 'BestDR5.QsoBest': '<urlopen error timed out>', 'BestDR5.QsoSpec': '<urlopen error timed out>', 'BestDR5.SpecPhoto': '<urlopen error timed out>', 'BestDR5.Unknown': '<urlopen error timed out>', 'BestDR5.RunQA': '<urlopen error timed out>', 'BestDR5.PhotoFamily': '<urlopen error timed out>', 'BestDR5.TiledTargetAll': '<urlopen error timed out>', 'BestDR5.Dependency': '<urlopen error timed out>', 'BestDR5.PhotoObj': '<urlopen error timed out>', 'BestDR5.InsideMask': '<urlopen error timed out>', 'BestDR5.Sector2Tile': '<urlopen error timed out>', 'BestDR7.SpecObjAll': '<urlopen error timed out>', 'BestDR7.TilingInfo': '<urlopen error timed out>', 'BestDR7.QsoBunch': '<urlopen error timed out>', 'BestDR7.Ap7Mag': '<urlopen error timed out>', 'BestDR7.SDSSConstants': '<urlopen error timed out>', 'BestDR7.TargPhotoObj': '<urlopen error timed out>', 'BestDR7.PubHistory': '<urlopen error timed out>', 'BestDR7.QsoConcordance': '<urlopen error timed out>', 'BestDR7.MaskType': '<urlopen error timed out>', 'BestDR7.ProfileDefs': '<urlopen error timed out>', 'BestDR7.Rosat': '<urlopen error timed out>', 'BestDR7.TiMask': '<urlopen error timed out>', 'BestDR7.ELRedShift': '<urlopen error timed out>', 'BestDR7.SecTarget': '<urlopen error timed out>', 'BestDR7.XCRedshift': '<urlopen error timed out>', 'BestDR7.ObjType': '<urlopen error timed out>', 'BestDR7.PhotoPrimary': '<urlopen error timed out>', 'BestDR7.History': '<urlopen error timed out>', 'BestDR7."Zone"': '<urlopen error timed out>', 'BestDR7.Sector': '<urlopen error timed out>', 'BestDR7.PhotoFlags': '<urlopen error timed out>', 'BestDR7.FieldQuality': '<urlopen error timed out>', 'BestDR7.SpecObj': '<urlopen error timed out>', 'BestDR7.PhotoSecondary': '<urlopen error timed out>', 'BestDR7.TilingRun': '<urlopen error timed out>', 'BestDR7.TilingMask': '<urlopen error timed out>', 'BestDR7.PhotoAux': '<urlopen error timed out>', 'BestDR7.TargPhotoSecondary': '<urlopen error timed out>', 'BestDR7.RunShift': '<urlopen error timed out>', 'BestDR7.FieldProfile': '<urlopen error timed out>', 'BestDR7.Frame': '<urlopen error timed out>', 'BestDR7.Region2Box': '<urlopen error timed out>', 'BestDR7.Sky': '<urlopen error timed out>', 'BestDR7.TilingBoundary': '<urlopen error timed out>', 'BestDR7.FramesStatus': '<urlopen error timed out>', 'BestDR7.StripeDefs': '<urlopen error timed out>', 'BestDR7.TableDesc': '<urlopen error timed out>', 'BestDR7.QsoTarget': '<urlopen error timed out>', 'BestDR7.UberAstro': '<urlopen error timed out>', 'BestDR7.FieldMask': '<urlopen error timed out>', 'BestDR7.TargetParam': '<urlopen error timed out>', 'BestDR7.QsoCatalog': '<urlopen error timed out>', 'BestDR7.Photoz': '<urlopen error timed out>', 'BestDR7."Match"': '<urlopen error timed out>', 'BestDR7.SiteConstants': '<urlopen error timed out>', 'BestDR7.PhotoMode': '<urlopen error timed out>', 'BestDR7."Region"': '<urlopen error timed out>', 'BestDR7.Chunk': '<urlopen error timed out>', 'BestDR7.SpecLine': '<urlopen error timed out>', 'BestDR7.CoordType': '<urlopen error timed out>', 'BestDR7.UberCalibStatus': '<urlopen error timed out>', 'BestDR7.SpecLineAll': '<urlopen error timed out>', 'BestDR7.StarTag': '<urlopen error timed out>', 'BestDR7.PspStatus': '<urlopen error timed out>', 'BestDR7.PsObjAll': '<urlopen error timed out>', 'BestDR7.BestTarget2Sector': '<urlopen error timed out>', 'BestDR7.Inventory': '<urlopen error timed out>', 'BestDR7.Rmatrix': '<urlopen error timed out>', 'BestDR7.TilingGeometry': '<urlopen error timed out>', 'BestDR7.Versions': '<urlopen error timed out>', 'BestDR7.Segment': '<urlopen error timed out>', 'BestDR7.LoadHistory': '<urlopen error timed out>', 'BestDR7.ProperMotions': '<urlopen error timed out>', 'BestDR7.IndexMap': '<urlopen error timed out>', 'BestDR7.Mask': '<urlopen error timed out>', 'BestDR7.Target': '<urlopen error timed out>', 'BestDR7.SpecPhotoAll': '<urlopen error timed out>', 'BestDR7.DBObjects': '<urlopen error timed out>', 'BestDR7.Neighbors': '<urlopen error timed out>', 'BestDR7.DBColumns': '<urlopen error timed out>', 'BestDR7.ObjMask': '<urlopen error timed out>', 'BestDR7.DR3QuasarCatalog': '<urlopen error timed out>', 'BestDR7.RecentQueries': '<urlopen error timed out>', 'BestDR7.PrimTarget': '<urlopen error timed out>', 'BestDR7.TargPhotoObjAll': '<urlopen error timed out>', 'BestDR7.Galaxy': '<urlopen error timed out>', 'BestDR7.MaskedObject': '<urlopen error timed out>', 'BestDR7.MatchHead': '<urlopen error timed out>', 'BestDR7.PhotoProfile': '<urlopen error timed out>', 'BestDR7.TargPhotoPrimary': '<urlopen error timed out>', 'BestDR7.USNO': '<urlopen error timed out>', 'BestDR7.TileAll': '<urlopen error timed out>', 'BestDR7.DBViewCols': '<urlopen error timed out>', 'BestDR7.PartitionMap': '<urlopen error timed out>', 'BestDR7.FileGroupMap': '<urlopen error timed out>', 'BestDR7.Glossary': '<urlopen error timed out>', 'BestDR7."Diagnostics"': '<urlopen error timed out>', 'BestDR7.Field': '<urlopen error timed out>', 'BestDR7.TargetInfo': '<urlopen error timed out>', 'BestDR7.DataConstants': '<urlopen error timed out>', 'BestDR7.TilingNote': '<urlopen error timed out>', 'BestDR7.sppParams': '<urlopen error timed out>', 'BestDR7.RegionArcs': '<urlopen error timed out>', 'BestDR7.Stetson': '<urlopen error timed out>', 'BestDR7.GalaxyTag': '<urlopen error timed out>', 'BestDR7.FieldQA': '<urlopen error timed out>', 'BestDR7.Run': '<urlopen error timed out>', 'BestDR7.PhotoAuxAll': '<urlopen error timed out>', 'BestDR7."First"': '<urlopen error timed out>', 'BestDR7.DR5QuasarCatalog': '<urlopen error timed out>', 'BestDR7.TargPhotoTag': '<urlopen error timed out>', 'BestDR7.RegionPatch': '<urlopen error timed out>', 'BestDR7.TargRunQA': '<urlopen error timed out>', 'BestDR7.spbsParams': '<urlopen error timed out>', 'BestDR7.HoleObj': '<urlopen error timed out>', 'BestDR7.SpecZStatus': '<urlopen error timed out>', 'BestDR7.PhotoTag': '<urlopen error timed out>', 'BestDR7.Columns': '<urlopen error timed out>', 'BestDR7.SpecLineNames': '<urlopen error timed out>', 'BestDR7.SiteDBs': '<urlopen error timed out>', 'BestDR7.TiledTarget': '<urlopen error timed out>', 'BestDR7.UberCal': '<urlopen error timed out>', 'BestDR7.Star': '<urlopen error timed out>', 'BestDR7.QueryResults': '<urlopen error timed out>', 'BestDR7.SpecZWarning': '<urlopen error timed out>', 'BestDR7.QsoCatalogAll': '<urlopen error timed out>', 'BestDR7.OrigField': '<urlopen error timed out>', 'BestDR7.RegionConvex': '<urlopen error timed out>', 'BestDR7.Algorithm': '<urlopen error timed out>', 'BestDR7.OrigPhotoObjAll': '<urlopen error timed out>', 'BestDR7.Photoz2': '<urlopen error timed out>', 'BestDR7.sppLines': '<urlopen error timed out>', 'BestDR7.SpecLineIndex': '<urlopen error timed out>', 'BestDR7.ImageMask': '<urlopen error timed out>', 'BestDR7.QsoConcordanceAll': '<urlopen error timed out>', 'BestDR7.ProgramType': '<urlopen error timed out>', 'BestDR7.SpecClass': '<urlopen error timed out>', 'BestDR7.PhotoObjAll': '<urlopen error timed out>', 'BestDR7.RC3': '<urlopen error timed out>', 'BestDR7.PhotoType': '<urlopen error timed out>', 'BestDR7.Tile': '<urlopen error timed out>', 'BestDR7.HalfSpace': '<urlopen error timed out>', 'BestDR7.PlateX': '<urlopen error timed out>', 'BestDR7.QsoBest': '<urlopen error timed out>', 'BestDR7.QsoSpec': '<urlopen error timed out>', 'BestDR7.SpecPhoto': '<urlopen error timed out>', 'BestDR7.Unknown': '<urlopen error timed out>', 'BestDR7.RunQA': '<urlopen error timed out>', 'BestDR7.PhotoFamily': '<urlopen error timed out>', 'BestDR7.TiledTargetAll': '<urlopen error timed out>', 'BestDR7.Dependency': '<urlopen error timed out>', 'BestDR7.PhotoObj': '<urlopen error timed out>', 'BestDR7.InsideMask': '<urlopen error timed out>', 'BestDR7.Sector2Tile': '<urlopen error timed out>', 'BestDR8.SpecObjAll': '<urlopen error timed out>', 'BestDR8.TwoMass': '<urlopen error timed out>', 'BestDR8.FieldMask': '<urlopen error timed out>', 'BestDR8.HoleType': '<urlopen error timed out>', 'BestDR8.PhotoStatus': '<urlopen error timed out>', 'BestDR8.SDSSConstants': '<urlopen error timed out>', 'BestDR8.galSpecIndx': '<urlopen error timed out>', 'BestDR8.ObjType': '<urlopen error timed out>', 'BestDR8.sdssTilingGeometry': '<urlopen error timed out>', 'BestDR8.PhotoPrimary': '<urlopen error timed out>', 'BestDR8.History': '<urlopen error timed out>', 'BestDR8.Plate2Target': '<urlopen error timed out>', 'BestDR8.detectionIndex': '<urlopen error timed out>', 'BestDR8.segue1SpecObjAll': '<urlopen error timed out>', 'BestDR8.thingIndex': '<urlopen error timed out>', 'BestDR8.Sector': '<urlopen error timed out>', 'BestDR8.PhotoFlags': '<urlopen error timed out>', 'BestDR8.FieldQuality': '<urlopen error timed out>', 'BestDR8.SpecObj': '<urlopen error timed out>', 'BestDR8.PhotoSecondary': '<urlopen error timed out>', 'BestDR8.RunShift': '<urlopen error timed out>', 'BestDR8.FieldProfile': '<urlopen error timed out>', 'BestDR8.zooSpec': '<urlopen error timed out>', 'BestDR8.Region2Box': '<urlopen error timed out>', 'BestDR8.Sky': '<urlopen error timed out>', 'BestDR8.TiMask': '<urlopen error timed out>', 'BestDR8.StripeDefs': '<urlopen error timed out>', 'BestDR8.sdssTilingBoundary': '<urlopen error timed out>', 'BestDR8.sdssTileAll': '<urlopen error timed out>', 'BestDR8.PhotoPrimaryDR7': '<urlopen error timed out>', 'BestDR8.segueTargetAll': '<urlopen error timed out>', 'BestDR8.Photoz': '<urlopen error timed out>', 'BestDR8.galSpecInfo': '<urlopen error timed out>', 'BestDR8.SiteConstants': '<urlopen error timed out>', 'BestDR8.PhotoMode': '<urlopen error timed out>', 'BestDR8."Region"': '<urlopen error timed out>', 'BestDR8.Frame': '<urlopen error timed out>', 'BestDR8.PhotozTemplateCoeff': '<urlopen error timed out>', 'BestDR8.CoordType': '<urlopen error timed out>', 'BestDR8.zooVotes': '<urlopen error timed out>', 'BestDR8.UberCalibStatus': '<urlopen error timed out>', 'BestDR8.zooConfidence': '<urlopen error timed out>', 'BestDR8.zooMirrorBias': '<urlopen error timed out>', 'BestDR8.PspStatus': '<urlopen error timed out>', 'BestDR8.Versions': '<urlopen error timed out>', 'BestDR8.StarTag': '<urlopen error timed out>', 'BestDR8.PhotoType': '<urlopen error timed out>', 'BestDR8.Inventory': '<urlopen error timed out>', 'BestDR8.Rmatrix': '<urlopen error timed out>', 'BestDR8.LoadHistory': '<urlopen error timed out>', 'BestDR8.PhotozRFTemplateCoeff': '<urlopen error timed out>', 'BestDR8.neighbors': '<urlopen error timed out>', 'BestDR8.sdssTilingMask': '<urlopen error timed out>', 'BestDR8.Mask': '<urlopen error timed out>', 'BestDR8.sdssTiledTargetAll': '<urlopen error timed out>', 'BestDR8.FramesStatus': '<urlopen error timed out>', 'BestDR8.Target': '<urlopen error timed out>', 'BestDR8.SpecPhotoAll': '<urlopen error timed out>', 'BestDR8.DBObjects': '<urlopen error timed out>', 'BestDR8.DBColumns': '<urlopen error timed out>', 'BestDR8.RecentQueries': '<urlopen error timed out>', 'BestDR8.PrimTarget': '<urlopen error timed out>', 'BestDR8.TwoMassXSC': '<urlopen error timed out>', 'BestDR8.Galaxy': '<urlopen error timed out>', 'BestDR8.MaskedObject': '<urlopen error timed out>', 'BestDR8.PhotoProfile': '<urlopen error timed out>', 'BestDR8.USNO': '<urlopen error timed out>', 'BestDR8.TableDesc': '<urlopen error timed out>', 'BestDR8.DBViewCols': '<urlopen error timed out>', 'BestDR8.ROSAT': '<urlopen error timed out>', 'BestDR8.FileGroupMap': '<urlopen error timed out>', 'BestDR8.sdssImagingHalfSpaces': '<urlopen error timed out>', 'BestDR8.sdssPolygon2Field': '<urlopen error timed out>', 'BestDR8."Diagnostics"': '<urlopen error timed out>', 'BestDR8.Field': '<urlopen error timed out>', 'BestDR8.TargetInfo': '<urlopen error timed out>', 'BestDR8.DataConstants': '<urlopen error timed out>', 'BestDR8.sdssTargetParam': '<urlopen error timed out>', 'BestDR8.RegionArcs': '<urlopen error timed out>', 'BestDR8.GalaxyTag': '<urlopen error timed out>', 'BestDR8.galSpecLine': '<urlopen error timed out>', 'BestDR8.Run': '<urlopen error timed out>', 'BestDR8.ProperMotions': '<urlopen error timed out>', 'BestDR8.galSpecExtra': '<urlopen error timed out>', 'BestDR8.RegionPatch': '<urlopen error timed out>', 'BestDR8.IndexMap': '<urlopen error timed out>', 'BestDR8.SpecZStatus': '<urlopen error timed out>', 'BestDR8.PhotoTag': '<urlopen error timed out>', 'BestDR8.Columns': '<urlopen error timed out>', 'BestDR8.SpecLineNames': '<urlopen error timed out>', 'BestDR8.SiteDBs': '<urlopen error timed out>', 'BestDR8.sdssPolygons': '<urlopen error timed out>', 'BestDR8.Star': '<urlopen error timed out>', 'BestDR8.QueryResults': '<urlopen error timed out>', 'BestDR8.SpecZWarning': '<urlopen error timed out>', 'BestDR8."FIRST"': '<urlopen error timed out>', 'BestDR8.RegionConvex': '<urlopen error timed out>', 'BestDR8.MaskType': '<urlopen error timed out>', 'BestDR8.segueSpecObjAll': '<urlopen error timed out>', 'BestDR8.sppLines': '<urlopen error timed out>', 'BestDR8."Zone"': '<urlopen error timed out>', 'BestDR8.ImageMask': '<urlopen error timed out>', 'BestDR8.ProgramType': '<urlopen error timed out>', 'BestDR8.SpecClass': '<urlopen error timed out>', 'BestDR8.PhotozRF': '<urlopen error timed out>', 'BestDR8.segue2SpecObjAll': '<urlopen error timed out>', 'BestDR8.sppParams': '<urlopen error timed out>', 'BestDR8.PhotoObjAll': '<urlopen error timed out>', 'BestDR8.RC3': '<urlopen error timed out>', 'BestDR8.PartitionMap': '<urlopen error timed out>', 'BestDR8.zooNoSpec': '<urlopen error timed out>', 'BestDR8.HalfSpace': '<urlopen error timed out>', 'BestDR8.PlateX': '<urlopen error timed out>', 'BestDR8.PhotoObjDR7': '<urlopen error timed out>', 'BestDR8.sdssTilingInfo': '<urlopen error timed out>', 'BestDR8.SpecPhoto': '<urlopen error timed out>', 'BestDR8.Unknown': '<urlopen error timed out>', 'BestDR8.zooMonochromeBias': '<urlopen error timed out>', 'BestDR8.PhotoFamily': '<urlopen error timed out>', 'BestDR8.Dependency': '<urlopen error timed out>'}
   6.141 +Table query completed after 10.012227535247803 seconds
   6.142 +
   6.143 +
   6.144 +
   6.145 +
   6.146 +## Restart Araybwyn VM, as it is not responding, and cannot be ssh'd into
   6.147 +
   6.148 +ssh trop02
   6.149 +
   6.150 +     ## Destroy VM
   6.151 +    vmname=Araybwyn
   6.152 +
   6.153 +    source "${HOME}/libvirt.settings"
   6.154 +    virsh \
   6.155 +        --connect "${connection}" \
   6.156 +         destroy \
   6.157 +            "${vmname:?}"
   6.158 +
   6.159 +   ## Startup VM
   6.160 +    vmname=Araybwyn
   6.161 +
   6.162 +    source "${HOME}/libvirt.settings"
   6.163 +    virsh \
   6.164 +        --connect "${connection}" \
   6.165 +         start \
   6.166 +            "${vmname:?}"
   6.167 +
   6.168 +
   6.169 +    ## Restart using compose
   6.170 +
   6.171 +	export buildtag=2.1.33
   6.172 +	source "${HOME:?}/chain.properties"
   6.173 +
   6.174 +	docker-compose --file "docker-compose.yml" run firethorn-py
   6.175 +
   6.176 +	Starting stevedore_jarmila_1  ... done
   6.177 +	Starting stevedore_bethany_1  ... done
   6.178 +	Starting stevedore_carolina_1 ... done
   6.179 +	Starting stevedore_gillian_1  ... done
   6.180 +	Python 3.5.2 (default, Nov 12 2018, 13:43:14) 
   6.181 +	[GCC 5.4.0 20160609] on linux
   6.182 +	Type "help", "copyright", "credits" or "license" for more information.
   6.183 +	>>> exit()
   6.184 +	[Stevedore@Araybwyn ~]$ docker ps
   6.185 +	CONTAINER ID        IMAGE                        COMMAND                  CREATED             STATUS                            PORTS                    NAMES
   6.186 +	44aaf5d76a44        firethorn/firethorn:2.1.33   "/bin/sh -c '/var/lo…"   6 days ago          Up 6 seconds (health: starting)   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   6.187 +	4ae7d97e1d09        firethorn/postgres:2.1.33    "docker-entrypoint.s…"   6 days ago          Up 38 seconds                     5432/tcp                 stevedore_carolina_1
   6.188 +	9099fe97de6d        firethorn/postgres:2.1.33    "docker-entrypoint.s…"   6 days ago          Up 38 seconds                     5432/tcp                 stevedore_bethany_1
   6.189 +	b75f992b86e7        firethorn/ogsadai:2.1.33     "/bin/sh -c '/var/lo…"   6 days ago          Up 38 seconds (healthy)           8080/tcp                 stevedore_jarmila_1
   6.190 +
   6.191 +
   6.192 +    ## curl http://tap.roe.ac.uk/firethorn/system/info still times out
   6.193 +    ## Looks like the Apache proxy service is not responding
   6.194 +    ## ssh   Stevedore@Acilamwen 
   6.195 +    ## docker restart apache
   6.196 +    ##  curl http://tap.roe.ac.uk/firethorn/system/info now works
   6.197 +
   6.198 +
   6.199 +
   6.200 +   ## Try a sync query:
   6.201 +   ## curl http://tap.roe.ac.uk/ssa/sync?REQUEST=doQuery&QUERY=SELECT+TOP+1+*+from+BestDR1.Sector&LANG=ADQL 
   6.202 +
   6.203 +
   6.204 +      <VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/v1.3" version="1.3">
   6.205 +<RESOURCE type="results">
   6.206 +<INFO name="QUERY_STATUS" value="OK"/>
   6.207 +<INFO name="link" value="http://tap.roe.ac.uk/firethorn/adql/table/29635605"/>
   6.208 +<TABLE ID="table.29635605" name="XX_YEXCG642EFIPCAAAAFVY6KMXB4">
   6.209 +<FIELD ID="column.29635659" name="regionID" datatype="int" arraysize="1" xtype="null">
   6.210 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635659"/>
   6.211 +</FIELD>
   6.212 +<FIELD ID="column.29635661" name="nTiles" datatype="int" arraysize="1" xtype="null">
   6.213 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635661"/>
   6.214 +</FIELD>
   6.215 +<FIELD ID="column.29635663" name="area" datatype="float" xtype="null">
   6.216 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635663"/>
   6.217 +</FIELD>
   6.218 +<DATA>
   6.219 +<TABLEDATA>
   6.220 +<TR>
   6.221 +<TD>26816</TD>
   6.222 +<TD>1</TD>
   6.223 +<TD>0.0</TD>
   6.224 +</TR>
   6.225 +</TABLEDATA>
   6.226 +</DATA>
   6.227 +</TABLE>
   6.228 +</RESOURCE>
   6.229 +</VOTABLE>
   6.230 +
   6.231 +
   6.232 +   ## Service has recovered
   6.233 +
   6.234 +
   6.235 +
     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/doc/notes/stv/20190626-Firethorn-Swarm.txt	Tue Jul 02 04:12:49 2019 +0200
     7.3 @@ -0,0 +1,426 @@
     7.4 +#
     7.5 +# <meta:header>
     7.6 +#   <meta:licence>
     7.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     7.8 +#
     7.9 +#     This information is free software: you can redistribute it and/or modify
    7.10 +#     it under the terms of the GNU General Public License as published by
    7.11 +#     the Free Software Foundation, either version 3 of the License, or
    7.12 +#     (at your option) any later version.
    7.13 +#
    7.14 +#     This information is distributed in the hope that it will be useful,
    7.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    7.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    7.17 +#     GNU General Public License for more details.
    7.18 +#  
    7.19 +#     You should have received a copy of the GNU General Public License
    7.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    7.21 +#   </meta:licence>
    7.22 +# </meta:header>
    7.23 +#
    7.24 +#
    7.25 +
    7.26 +
    7.27 +## ------------------------------------------------------------------------------------------
    7.28 +## Create Master VM 
    7.29 +## ------------------------------------------------------------------------------------------
    7.30 +
    7.31 +
    7.32 +createvm Cadelicia (master)
    7.33 +..
    7.34 +Domain Cadelicia started
    7.35 +
    7.36 +ssh Stevedore@Cadelicia
    7.37 +
    7.38 +
    7.39 +
    7.40 +## ------------------------------------------------------------------------------------------
    7.41 +## Initialize Swarm with VM's address
    7.42 +## ------------------------------------------------------------------------------------------
    7.43 +
    7.44 +[Stevedore@Cadelicia ~]$ ifconfig
    7.45 +
    7.46 +       ..  192.168.201.11  ..
    7.47 +
    7.48 +
    7.49 +docker swarm init --advertise-addr 192.168.201.11  
    7.50 +
    7.51 +
    7.52 +		Swarm initialized: current node (5zrs2po0x1teeyzigf09mw5vi) is now a manager.
    7.53 +
    7.54 +		To add a worker to this swarm, run the following command:
    7.55 +
    7.56 +		    docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
    7.57 +
    7.58 +		To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    7.59 +
    7.60 +
    7.61 +## ------------------------------------------------------------------------------------------
    7.62 +## Create Spark-net network
    7.63 +## ------------------------------------------------------------------------------------------
    7.64 +## docker network create --driver overlay firethorn-net
    7.65 +
    7.66 +
    7.67 +    ## Fix Firewall issue
    7.68 +    sudo su
    7.69 +	    firewall-cmd --add-port=2377/tcp --permanent
    7.70 +	    firewall-cmd --add-port=2377/tcp --permanent
    7.71 +	    firewall-cmd --add-port=7946/tcp --permanent
    7.72 +	    firewall-cmd --add-port=4789/tcp --permanent
    7.73 +	    firewall-cmd --add-port=4789/udp --permanent
    7.74 +	    iptables -A INPUT -p 50 -j ACCEPT    
    7.75 +	    firewall-cmd --reload
    7.76 +    exit
    7.77 +
    7.78 +
    7.79 +## -------------------------------------------------------------------------------------------
    7.80 +## Add secret function.
    7.81 +## -------------------------------------------------------------------------------------------
    7.82 +
    7.83 +    secrethost='stv@shepseskaf.roe.ac.uk'
    7.84 +    secretfile='${HOME:?}/secret.store.20181212'
    7.85 +
    7.86 +    secret()
    7.87 +    {
    7.88 +        local key=${1:?}
    7.89 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    7.90 +    }
    7.91 +
    7.92 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    7.93 +
    7.94 +
    7.95 +
    7.96 +## -------------------------------------------------------------------------------------------
    7.97 +## Set the target branch and builtag
    7.98 +## -------------------------------------------------------------------------------------------
    7.99 +
   7.100 +
   7.101 +    export buildtag=2.1.33
   7.102 +    export branch=default
   7.103 +
   7.104 +
   7.105 +
   7.106 +## -------------------------------------------------------------------------------------------
   7.107 +## Create our chain.properties file
   7.108 +## -------------------------------------------------------------------------------------------
   7.109 +
   7.110 +cat > "${HOME:?}/chain.properties" << EOF
   7.111 +
   7.112 +metadata=postgres
   7.113 +metauser=$(pwgen 20 1)
   7.114 +metapass=$(pwgen 20 1) 
   7.115 +
   7.116 +userhost=$(secret 'firethorn.user.host')
   7.117 +userdata=$(secret 'firethorn.user.data')
   7.118 +useruser=$(secret 'firethorn.user.user')
   7.119 +userpass=$(secret 'firethorn.user.pass')
   7.120 +usertype=mssql
   7.121 +
   7.122 +datahost=$(secret 'firethorn.data.host')
   7.123 +datadata=$(secret 'firethorn.data.data')
   7.124 +datacatalog=$(secret 'firethorn.data.catalog')
   7.125 +datatype=$(secret 'firethorn.data.type')
   7.126 +datauser=$(secret 'firethorn.data.user')
   7.127 +datapass=$(secret 'firethorn.data.pass')
   7.128 +
   7.129 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   7.130 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   7.131 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   7.132 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   7.133 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   7.134 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   7.135 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   7.136 +
   7.137 +tunneluser=$(secret 'ssh.tunnel.user')
   7.138 +tunnelhost=$(secret 'ssh.tunnel.host')
   7.139 +
   7.140 +admingroup=$(pwgen 20 1)	
   7.141 +adminuser=$(pwgen 20 1)
   7.142 +adminpass=$(pwgen 20 1)
   7.143 +
   7.144 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   7.145 + 
   7.146 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   7.147 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   7.148 +
   7.149 +guestgroup=friends
   7.150 +endpoint=$(secret 'endpoint')
   7.151 +
   7.152 +EOF
   7.153 +
   7.154 +
   7.155 +
   7.156 +## -----------------------------------------------------
   7.157 +## Create our Firethorn properties.
   7.158 +## -----------------------------------------------------
   7.159 +
   7.160 +source "${HOME:?}/chain.properties"
   7.161 +cat > "${HOME:?}/firethorn.properties" << EOF
   7.162 +
   7.163 +firethorn.meta.type=pgsql
   7.164 +firethorn.meta.driver=org.postgresql.Driver
   7.165 +firethorn.meta.host=bethany
   7.166 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   7.167 +firethorn.meta.data=postgres
   7.168 +firethorn.meta.user=${metauser:?}
   7.169 +firethorn.meta.pass=${metapass:?}
   7.170 +
   7.171 +firethorn.user.type=mssql
   7.172 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   7.173 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   7.174 +firethorn.user.host=${userhost:?}
   7.175 +firethorn.user.data=${userdata:?}
   7.176 +firethorn.user.user=${useruser:?}
   7.177 +firethorn.user.pass=${userpass:?}
   7.178 +
   7.179 +firethorn.tapschema.resource.name=${tapschemajdbc}
   7.180 +firethorn.tapschema.database.name=${tapschemadata}
   7.181 +firethorn.tapschema.database.host=${tapschemahost}
   7.182 +firethorn.tapschema.database.port=${tapschemaport}
   7.183 +firethorn.tapschema.database.user=${tapschemauser}
   7.184 +firethorn.tapschema.database.pass=${tapschemapass}
   7.185 +
   7.186 +firethorn.admin.community=${admingroup:?}
   7.187 +firethorn.admin.user.name=${adminuser:?}
   7.188 +firethorn.admin.user.pass=${adminpass:?}
   7.189 +
   7.190 +firethorn.guest.community=${guestgroup:?}
   7.191 +
   7.192 +adql.query.delay.first=${adql_query_delay_first:?}
   7.193 +
   7.194 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   7.195 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   7.196 +	
   7.197 +
   7.198 +EOF
   7.199 +
   7.200 +
   7.201 +
   7.202 +## -------------------------------------------------------------------------------------------
   7.203 +## Create our compose env files.
   7.204 +## -------------------------------------------------------------------------------------------
   7.205 +
   7.206 +source "${HOME:?}/chain.properties"
   7.207 +cat > "${HOME:?}/.env" << EOF
   7.208 +
   7.209 +buildtag=${branch:?}
   7.210 +endpoint=${endpoint:?}
   7.211 +
   7.212 +
   7.213 +metadata=postgres
   7.214 +metauser=${metauser:?}
   7.215 +metapass=${metapass:?}
   7.216 +firethorn.meta.type=pgsql
   7.217 +firethorn.meta.driver=org.postgresql.Driver
   7.218 +firethorn.meta.host=bethany
   7.219 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   7.220 +firethorn.meta.data=postgres
   7.221 +firethorn.meta.user=${metauser:?}
   7.222 +firethorn.meta.pass=${metapass:?}
   7.223 +
   7.224 +
   7.225 +datadata=${datadata:?}
   7.226 +datauser=${datauser:?}
   7.227 +datapass=${datapass:?}
   7.228 +datahost=${datahost:?}
   7.229 +datacatalog=${datacatalog:?}
   7.230 +datatype=mssql
   7.231 +
   7.232 +
   7.233 +tapschemadata=${tapschemadata:?}
   7.234 +tapschemauser=${tapschemauser:?}
   7.235 +tapschemapass=${tapschemapass:?}
   7.236 +tapschemahost=${tapschemahost}
   7.237 +tapschemaport=${tapschemaport:?}
   7.238 +tapschemajdbc=${tapschemajdbc:?}
   7.239 +tapschematype=${tapschematype:?}
   7.240 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   7.241 +firethorn.tapschema.database.name=${tapschemadata:?}
   7.242 +firethorn.tapschema.database.host=${tapschemahost:?}
   7.243 +firethorn.tapschema.database.port=5432
   7.244 +firethorn.tapschema.database.user=${tapschemauser:?}
   7.245 +firethorn.tapschema.database.pass=${tapschemapass:?}
   7.246 +firethorn.tapschema.database.type=pgsql
   7.247 +
   7.248 +userhost=${userhost:?}
   7.249 +userdata=${userdata:?}
   7.250 +useruser=${useruser:?}
   7.251 +userpass=${userpass:?}
   7.252 +usertype=mssql
   7.253 +firethorn.user.type=mssql
   7.254 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   7.255 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   7.256 +firethorn.user.host=${userhost:?}
   7.257 +firethorn.user.data=${userdata:?}
   7.258 +firethorn.user.user=${useruser:?}
   7.259 +firethorn.user.pass=${userpass:?}
   7.260 +
   7.261 +
   7.262 +admingroup=${admingroup:?}
   7.263 +adminuser=${adminuser:?}
   7.264 +adminpass=${adminpass:?}
   7.265 +guestgroup=${guestgroup:?}
   7.266 +firethorn.admin.community=${admingroup:?}
   7.267 +firethorn.admin.user.name=${adminuser:?}
   7.268 +firethorn.admin.user.pass=${adminpass:?}
   7.269 +firethorn.guest.community=${guestgroup:?}
   7.270 +
   7.271 +
   7.272 +adql_query_delay_first=${adql_query_delay_first:?}
   7.273 +adql.query.delay.first=${adql_query_delay_first:?}
   7.274 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   7.275 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   7.276 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   7.277 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   7.278 +
   7.279 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   7.280 +
   7.281 +
   7.282 +EOF
   7.283 +
   7.284 +
   7.285 +source "${HOME:?}/chain.properties"
   7.286 +cat > "${HOME:?}/.tapschema_env" << EOF
   7.287 +
   7.288 +POSTGRES_DB=${tapschemadatabase}
   7.289 +POSTGRES_USER=${tapschemauser}
   7.290 +POSTGRES_PASSWORD=${tapschemapass}
   7.291 +
   7.292 +EOF
   7.293 +
   7.294 +
   7.295 +source "${HOME:?}/chain.properties"
   7.296 +cat > "${HOME:?}/.metadata_env" << EOF
   7.297 +
   7.298 +POSTGRES_DB=${metadata}
   7.299 +POSTGRES_USER=${metauser}
   7.300 +POSTGRES_PASSWORD=${metapass}
   7.301 +         
   7.302 +EOF
   7.303 +
   7.304 +
   7.305 +## -------------------------------------------------------------------------------------------
   7.306 +## Fetch the TAP Deployer
   7.307 +## -------------------------------------------------------------------------------------------
   7.308 +
   7.309 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   7.310 +
   7.311 +
   7.312 +
   7.313 +## ------------------------------------------------------------------------
   7.314 +## Deploy Swarm stack
   7.315 +## ------------------------------------------------------------------------
   7.316 +
   7.317 +docker stack deploy -c docker-compose.yml ft
   7.318 +        .. 
   7.319 +	Creating service ft_bethany
   7.320 +	Creating service ft_carolina
   7.321 +	Creating service ft_jarmila
   7.322 +	Creating service ft_gillian
   7.323 +	Creating service ft_firethorn-py
   7.324 +     
   7.325 +
   7.326 +docker ps
   7.327 +
   7.328 +	CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                             PORTS               NAMES
   7.329 +	212df08e73b2        firethorn/firethorn-py:latest   "python3"                29 seconds ago      Up 27 seconds                                          ft_firethorn-py.1.rz43gsoc11ky3ldutdyq8d6y1
   7.330 +	028e656c70d1        firethorn/firethorn:latest      "/bin/sh -c '/var/lo…"   31 seconds ago      Up 27 seconds (health: starting)   8080/tcp            ft_gillian.1.u7plhcroj56qj7ajyd8eceh0j
   7.331 +	2b1766b8b1ad        firethorn/ogsadai:latest        "/bin/sh -c '/var/lo…"   34 seconds ago      Up 28 seconds (health: starting)   8080/tcp            ft_jarmila.1.x9h7g1pdiznlwxkqtelllr9os
   7.332 +	9ce64b692f6a        firethorn/postgres:latest       "docker-entrypoint.s…"   36 seconds ago      Up 28 seconds                      5432/tcp            ft_carolina.1.iw9fi4qbogkbej8yp2d9ktleo
   7.333 +	04187bab3f02        firethorn/postgres:latest       "docker-entrypoint.s…"   38 seconds ago      Up 37 seconds                      5432/tcp            ft_bethany.1.bcezzygrqrnj5mvwg5yfug9zf
   7.334 +
   7.335 +
   7.336 +docker service ls
   7.337 +
   7.338 +	ID                  NAME                MODE                REPLICAS            IMAGE                           PORTS
   7.339 +	srrn94ebri8c        ft_bethany          replicated          1/1                 firethorn/postgres:latest       
   7.340 +	7keylnyfy68h        ft_carolina         replicated          1/1                 firethorn/postgres:latest       
   7.341 +	5slvh38gsa7h        ft_firethorn-py     replicated          1/1                 firethorn/firethorn-py:latest   
   7.342 +	z0doy33yeok9        ft_gillian          replicated          1/1                 firethorn/firethorn:latest      *:8080->8080/tcp
   7.343 +	yewb5qwddzbc        ft_jarmila          replicated          1/1                 firethorn/ogsadai:latest       
   7.344 +
   7.345 +
   7.346 +
   7.347 +## ------------------------------------------------------------------------
   7.348 +## To remove all docker services 
   7.349 +## ------------------------------------------------------------------------
   7.350 +
   7.351 +## docker service rm $(docker service ls -q)
   7.352 +
   7.353 +
   7.354 +
   7.355 +## ------------------------------------------------------------------------
   7.356 +## To remove stack of services
   7.357 +## ------------------------------------------------------------------------
   7.358 +
   7.359 +## docker stack rm ft
   7.360 +
   7.361 +
   7.362 +
   7.363 +## ------------------------------------------------------------------------
   7.364 +## Run Docker Compose tap-deployer.yml from firethorn.py
   7.365 +## ------------------------------------------------------------------------
   7.366 +
   7.367 +
   7.368 +source "${HOME:?}/chain.properties"
   7.369 +
   7.370 +docker-compose --file "docker-compose.yml" run firethorn-py
   7.371 +
   7.372 +import firethorn
   7.373 +import firethorn_utils.configurator as configur
   7.374 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   7.375 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   7.376 +configurator =  configur.Configurator(ft)
   7.377 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   7.378 +
   7.379 +..
   7.380 +Importing JDBC Schema: ATLASDR1
   7.381 +Importing JDBC Schema: ATLASDR2
   7.382 +Importing JDBC Schema: ATLASDR3
   7.383 +Importing JDBC Schema: 2MASS
   7.384 +Importing JDBC Schema: 2XMM
   7.385 +Importing JDBC Schema: WISE
   7.386 +Importing JDBC Schema: VIKINGDR3
   7.387 +Importing JDBC Schema: VHSDR1
   7.388 +Importing JDBC Schema: MGC
   7.389 +Importing JDBC Schema: GLIMPSE
   7.390 +Importing JDBC Schema: FIRST
   7.391 +Importing JDBC Schema: DENIS
   7.392 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   7.393 +
   7.394 +..
   7.395 +
   7.396 +## ------------------------------------------------------------------------
   7.397 +## [OPTIONAL] Adding a second VM to the Swarm as a Worker node
   7.398 +## ------------------------------------------------------------------------
   7.399 +
   7.400 +createvm Erennon (worker)
   7.401 +Domain Erennon started
   7.402 +
   7.403 +ssh Stevedore@Erennon
   7.404 +
   7.405 +     docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
   7.406 +      ...
   7.407 +      This node joined a swarm as a worker.
   7.408 +      ...
   7.409 +
   7.410 +    
   7.411 +     ## Fix Firewall issue
   7.412 +     sudo su
   7.413 +     firewall-cmd --add-port=2377/tcp --permanent
   7.414 +     firewall-cmd --add-port=2377/tcp --permanent
   7.415 +     firewall-cmd --add-port=7946/tcp --permanent
   7.416 +     firewall-cmd --add-port=4789/udp --permanent
   7.417 +     firewall-cmd --add-port=4789/tcp --permanent
   7.418 +     iptables -A INPUT -p 50 -j ACCEPT    
   7.419 +     firewall-cmd --reload
   7.420 +     exit
   7.421 +
   7.422 +
   7.423 +## ------------------------------------------------------------------------
   7.424 +##  Scaling: If we were to scale the cluster, we Could do so as:
   7.425 +## ------------------------------------------------------------------------
   7.426 +docker service scale ft_gillian=4
   7.427 +docker service scale ft_jarmila=4
   7.428 +
   7.429 +## But this will not do anything at the moment
     8.1 --- a/doc/notes/zrq/20190508-01-vmdown-debug.txt	Tue Jul 02 04:12:16 2019 +0200
     8.2 +++ b/doc/notes/zrq/20190508-01-vmdown-debug.txt	Tue Jul 02 04:12:49 2019 +0200
     8.3 @@ -26,18 +26,16 @@
     8.4  
     8.5      docker ps -a
     8.6  
     8.7 ---START--
     8.8 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
     8.9 -16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                12 days ago         Exited (137) 12 days ago                              stevedore_firethorn-py_run_5
    8.10 -2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 12 days ago                              stevedore_firethorn-py_run_4
    8.11 -659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
    8.12 -cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
    8.13 -ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
    8.14 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
    8.15 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   8080/tcp                 stevedore_jarmila_1
    8.16 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_carolina_1
    8.17 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_bethany_1
    8.18 ---END--
    8.19 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
    8.20 +    >   16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                12 days ago         Exited (137) 12 days ago                              stevedore_firethorn-py_run_5
    8.21 +    >   2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 12 days ago                              stevedore_firethorn-py_run_4
    8.22 +    >   659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
    8.23 +    >   cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
    8.24 +    >   ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
    8.25 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
    8.26 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   8080/tcp                 stevedore_jarmila_1
    8.27 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_carolina_1
    8.28 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_bethany_1
    8.29  
    8.30  
    8.31  # -----------------------------------------------------
    8.32 @@ -50,15 +48,13 @@
    8.33  
    8.34      docker-compose --file "docker-compose.yml" run firethorn-py
    8.35  
    8.36 ---START--
    8.37 -Starting stevedore_bethany_1  ... done
    8.38 -Starting stevedore_jarmila_1  ... done
    8.39 -Starting stevedore_carolina_1 ... done
    8.40 -Starting stevedore_gillian_1  ... done
    8.41 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    8.42 -[GCC 5.4.0 20160609] on linux
    8.43 -Type "help", "copyright", "credits" or "license" for more information.
    8.44 ---END--
    8.45 +    >   Starting stevedore_bethany_1  ... done
    8.46 +    >   Starting stevedore_jarmila_1  ... done
    8.47 +    >   Starting stevedore_carolina_1 ... done
    8.48 +    >   Starting stevedore_gillian_1  ... done
    8.49 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    8.50 +    >   [GCC 5.4.0 20160609] on linux
    8.51 +    >   Type "help", "copyright", "credits" or "license" for more information.
    8.52  
    8.53  
    8.54  # -----------------------------------------------------
    8.55 @@ -167,167 +163,165 @@
    8.56  # *lots* of broken :-(
    8.57  #
    8.58  
    8.59 ---START--
    8.60 -{
    8.61 -  "resources": [],
    8.62 -  "owner": "http://gillian:8080/firethorn/community-member/57698559",
    8.63 -  "text": null,
    8.64 -  "callback": "http://gillian:8081/firethorn/callback/57698652",
    8.65 -  "modified": "2019-05-08T19:34:59.347",
    8.66 -  "history": [],
    8.67 -  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
    8.68 -  "delays": {
    8.69 -    "first": null,
    8.70 -    "last": null,
    8.71 -    "every": null
    8.72 -  },
    8.73 -  "url": "http://gillian:8080/firethorn/blue/query/57698652",
    8.74 -  "ident": "57698652",
    8.75 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
    8.76 -  "columns": [],
    8.77 -  "fields": [],
    8.78 -  "tables": [],
    8.79 -  "mode": "AUTO",
    8.80 -  "created": "2019-05-08T19:34:59.210",
    8.81 -  "name": "XX_5U5BMFLXCC4E4AAAAFVJR4I3RI",
    8.82 -  "results": {
    8.83 -    "count": 0,
    8.84 -    "state": "NONE",
    8.85 -    "formats": {
    8.86 -      "datatable": null,
    8.87 -      "votable": null
    8.88 -    },
    8.89 -    "table": null
    8.90 -  },
    8.91 -  "osql": null,
    8.92 -  "adql": null,
    8.93 -  "workspace": "http://gillian:8080/firethorn/adql/resource/57688319",
    8.94 -  "limits": {
    8.95 -    "cells": null,
    8.96 -    "rows": 1000000,
    8.97 -    "time": null
    8.98 -  },
    8.99 -  "resource": "http://gillian:8080/firethorn/adql/resource/57688319",
   8.100 -  "status": "FAILED",
   8.101 -  "syntax": {
   8.102 -    "status": "PARSE_ERROR",
   8.103 -    "friendly": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !",
   8.104 -    "message": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !"
   8.105 -  },
   8.106 -  "self": "http://gillian:8080/firethorn/blue/query/57698652"
   8.107 -}
   8.108 ->>> print(
   8.109 -...     query_obj.table()
   8.110 -...     )
   8.111 -ERROR:root:unknown url type: 'None'
   8.112 -Traceback (most recent call last):
   8.113 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.114 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.115 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.116 -    self.full_url = url
   8.117 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.118 -    self._parse()
   8.119 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.120 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.121 -ValueError: unknown url type: 'None'
   8.122 -ERROR:root:unknown url type: 'None'
   8.123 -Traceback (most recent call last):
   8.124 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.125 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.126 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.127 -    self.full_url = url
   8.128 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.129 -    self._parse()
   8.130 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.131 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.132 -ValueError: unknown url type: 'None'
   8.133 -ERROR:root:unknown url type: 'None'
   8.134 -Traceback (most recent call last):
   8.135 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.136 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.137 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.138 -    self.full_url = url
   8.139 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.140 -    self._parse()
   8.141 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.142 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.143 -ValueError: unknown url type: 'None'
   8.144 -null
   8.145 ->>> print(
   8.146 -...     query_obj.table().count()
   8.147 -...     )
   8.148 -ERROR:root:unknown url type: 'None'
   8.149 -Traceback (most recent call last):
   8.150 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.151 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.152 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.153 -    self.full_url = url
   8.154 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.155 -    self._parse()
   8.156 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.157 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.158 -ValueError: unknown url type: 'None'
   8.159 -ERROR:root:unknown url type: 'None'
   8.160 -Traceback (most recent call last):
   8.161 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.162 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.163 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.164 -    self.full_url = url
   8.165 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.166 -    self._parse()
   8.167 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.168 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.169 -ValueError: unknown url type: 'None'
   8.170 -ERROR:root:unknown url type: 'None'
   8.171 -Traceback (most recent call last):
   8.172 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.173 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.174 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.175 -    self.full_url = url
   8.176 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.177 -    self._parse()
   8.178 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.179 -    raise ValueError("unknown url type: %r" % self.full_url)
   8.180 -ValueError: unknown url type: 'None'
   8.181 -None
   8.182 ->>>
   8.183 ->>>
   8.184 ->>>
   8.185 ->>>
   8.186 ->>>
   8.187 ->>> resource = firethorn.firethorn_engine.select_adql_resources()[28]
   8.188 -ERROR:root:IncompleteRead(9713032 bytes read)
   8.189 -Traceback (most recent call last):
   8.190 -  File "/usr/lib/python3.5/http/client.py", line 541, in _get_chunk_left
   8.191 -    chunk_left = self._read_next_chunk_size()
   8.192 -  File "/usr/lib/python3.5/http/client.py", line 508, in _read_next_chunk_size
   8.193 -    return int(line, 16)
   8.194 -ValueError: invalid literal for int() with base 16: b''
   8.195 -
   8.196 -During handling of the above exception, another exception occurred:
   8.197 -
   8.198 -Traceback (most recent call last):
   8.199 -  File "/usr/lib/python3.5/http/client.py", line 558, in _readall_chunked
   8.200 -    chunk_left = self._get_chunk_left()
   8.201 -  File "/usr/lib/python3.5/http/client.py", line 543, in _get_chunk_left
   8.202 -    raise IncompleteRead(b'')
   8.203 -http.client.IncompleteRead: IncompleteRead(0 bytes read)
   8.204 -
   8.205 -During handling of the above exception, another exception occurred:
   8.206 -
   8.207 -Traceback (most recent call last):
   8.208 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/core/firethorn_engine.py", line 238, in select_adql_resources
   8.209 -    adqlresources =  json.loads(response.read().decode('utf-8'))
   8.210 -  File "/usr/lib/python3.5/http/client.py", line 455, in read
   8.211 -    return self._readall_chunked()
   8.212 -  File "/usr/lib/python3.5/http/client.py", line 565, in _readall_chunked
   8.213 -    raise IncompleteRead(b''.join(value))
   8.214 -http.client.IncompleteRead: IncompleteRead(9713032 bytes read)
   8.215 -Traceback (most recent call last):
   8.216 -  File "<stdin>", line 1, in <module>
   8.217 -IndexError: list index out of range
   8.218 ->>>
   8.219 ---END--
   8.220 +    >   {
   8.221 +    >     "resources": [],
   8.222 +    >     "owner": "http://gillian:8080/firethorn/community-member/57698559",
   8.223 +    >     "text": null,
   8.224 +    >     "callback": "http://gillian:8081/firethorn/callback/57698652",
   8.225 +    >     "modified": "2019-05-08T19:34:59.347",
   8.226 +    >     "history": [],
   8.227 +    >     "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
   8.228 +    >     "delays": {
   8.229 +    >       "first": null,
   8.230 +    >       "last": null,
   8.231 +    >       "every": null
   8.232 +    >     },
   8.233 +    >     "url": "http://gillian:8080/firethorn/blue/query/57698652",
   8.234 +    >     "ident": "57698652",
   8.235 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
   8.236 +    >     "columns": [],
   8.237 +    >     "fields": [],
   8.238 +    >     "tables": [],
   8.239 +    >     "mode": "AUTO",
   8.240 +    >     "created": "2019-05-08T19:34:59.210",
   8.241 +    >     "name": "XX_5U5BMFLXCC4E4AAAAFVJR4I3RI",
   8.242 +    >     "results": {
   8.243 +    >       "count": 0,
   8.244 +    >       "state": "NONE",
   8.245 +    >       "formats": {
   8.246 +    >         "datatable": null,
   8.247 +    >         "votable": null
   8.248 +    >       },
   8.249 +    >       "table": null
   8.250 +    >     },
   8.251 +    >     "osql": null,
   8.252 +    >     "adql": null,
   8.253 +    >     "workspace": "http://gillian:8080/firethorn/adql/resource/57688319",
   8.254 +    >     "limits": {
   8.255 +    >       "cells": null,
   8.256 +    >       "rows": 1000000,
   8.257 +    >       "time": null
   8.258 +    >     },
   8.259 +    >     "resource": "http://gillian:8080/firethorn/adql/resource/57688319",
   8.260 +    >     "status": "FAILED",
   8.261 +    >     "syntax": {
   8.262 +    >       "status": "PARSE_ERROR",
   8.263 +    >       "friendly": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !",
   8.264 +    >       "message": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !"
   8.265 +    >     },
   8.266 +    >     "self": "http://gillian:8080/firethorn/blue/query/57698652"
   8.267 +    >   }
   8.268 +    >   >>> print(
   8.269 +    >   ...     query_obj.table()
   8.270 +    >   ...     )
   8.271 +    >   ERROR:root:unknown url type: 'None'
   8.272 +    >   Traceback (most recent call last):
   8.273 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.274 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.275 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.276 +    >       self.full_url = url
   8.277 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.278 +    >       self._parse()
   8.279 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.280 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.281 +    >   ValueError: unknown url type: 'None'
   8.282 +    >   ERROR:root:unknown url type: 'None'
   8.283 +    >   Traceback (most recent call last):
   8.284 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.285 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.286 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.287 +    >       self.full_url = url
   8.288 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.289 +    >       self._parse()
   8.290 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.291 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.292 +    >   ValueError: unknown url type: 'None'
   8.293 +    >   ERROR:root:unknown url type: 'None'
   8.294 +    >   Traceback (most recent call last):
   8.295 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.296 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.297 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.298 +    >       self.full_url = url
   8.299 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.300 +    >       self._parse()
   8.301 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.302 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.303 +    >   ValueError: unknown url type: 'None'
   8.304 +    >   null
   8.305 +    >   >>> print(
   8.306 +    >   ...     query_obj.table().count()
   8.307 +    >   ...     )
   8.308 +    >   ERROR:root:unknown url type: 'None'
   8.309 +    >   Traceback (most recent call last):
   8.310 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.311 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.312 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.313 +    >       self.full_url = url
   8.314 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.315 +    >       self._parse()
   8.316 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.317 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.318 +    >   ValueError: unknown url type: 'None'
   8.319 +    >   ERROR:root:unknown url type: 'None'
   8.320 +    >   Traceback (most recent call last):
   8.321 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.322 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.323 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.324 +    >       self.full_url = url
   8.325 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.326 +    >       self._parse()
   8.327 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.328 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.329 +    >   ValueError: unknown url type: 'None'
   8.330 +    >   ERROR:root:unknown url type: 'None'
   8.331 +    >   Traceback (most recent call last):
   8.332 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   8.333 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   8.334 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   8.335 +    >       self.full_url = url
   8.336 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   8.337 +    >       self._parse()
   8.338 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   8.339 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   8.340 +    >   ValueError: unknown url type: 'None'
   8.341 +    >   None
   8.342 +    >   >>>
   8.343 +    >   >>>
   8.344 +    >   >>>
   8.345 +    >   >>>
   8.346 +    >   >>>
   8.347 +    >   >>> resource = firethorn.firethorn_engine.select_adql_resources()[28]
   8.348 +    >   ERROR:root:IncompleteRead(9713032 bytes read)
   8.349 +    >   Traceback (most recent call last):
   8.350 +    >     File "/usr/lib/python3.5/http/client.py", line 541, in _get_chunk_left
   8.351 +    >       chunk_left = self._read_next_chunk_size()
   8.352 +    >     File "/usr/lib/python3.5/http/client.py", line 508, in _read_next_chunk_size
   8.353 +    >       return int(line, 16)
   8.354 +    >   ValueError: invalid literal for int() with base 16: b''
   8.355 +    >   
   8.356 +    >   During handling of the above exception, another exception occurred:
   8.357 +    >   
   8.358 +    >   Traceback (most recent call last):
   8.359 +    >     File "/usr/lib/python3.5/http/client.py", line 558, in _readall_chunked
   8.360 +    >       chunk_left = self._get_chunk_left()
   8.361 +    >     File "/usr/lib/python3.5/http/client.py", line 543, in _get_chunk_left
   8.362 +    >       raise IncompleteRead(b'')
   8.363 +    >   http.client.IncompleteRead: IncompleteRead(0 bytes read)
   8.364 +    >   
   8.365 +    >   During handling of the above exception, another exception occurred:
   8.366 +    >   
   8.367 +    >   Traceback (most recent call last):
   8.368 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/core/firethorn_engine.py", line 238, in select_adql_resources
   8.369 +    >       adqlresources =  json.loads(response.read().decode('utf-8'))
   8.370 +    >     File "/usr/lib/python3.5/http/client.py", line 455, in read
   8.371 +    >       return self._readall_chunked()
   8.372 +    >     File "/usr/lib/python3.5/http/client.py", line 565, in _readall_chunked
   8.373 +    >       raise IncompleteRead(b''.join(value))
   8.374 +    >   http.client.IncompleteRead: IncompleteRead(9713032 bytes read)
   8.375 +    >   Traceback (most recent call last):
   8.376 +    >     File "<stdin>", line 1, in <module>
   8.377 +    >   IndexError: list index out of range
   8.378 +    >   >>>
   8.379  
   8.380  
   8.381  
   8.382 @@ -339,212 +333,208 @@
   8.383  
   8.384          tail -n 100 logs/firethorn-debug.log
   8.385  
   8.386 ---START--
   8.387 -2019-05-08 19:48:30,592 DEBUG [callback-interface-1] [AnonymousAuthenticator] Adding anonymous Authentication
   8.388 -2019-05-08 19:48:31,322 DEBUG [callback-interface-5] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.389 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   queryString [null]
   8.390 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   authType    [null]
   8.391 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.392 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug] Request headers
   8.393 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   [host][localhost:8081]
   8.394 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   authType    [null]
   8.395 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.396 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request headers
   8.397 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [host][localhost:8081]
   8.398 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.399 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [accept][*/*]
   8.400 -2019-05-08 19:48:32,659 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.401 -2019-05-08 19:48:34,236 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.402 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request parameters
   8.403 -2019-05-08 19:48:34,236 DEBUG [callback-interface-5] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.404 -2019-05-08 19:48:36,790 DEBUG [callback-interface-3] [HttpRequestDebug] ----
   8.405 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug]   [accept][*/*]
   8.406 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] Request parameters
   8.407 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] ----
   8.408 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug] Request properties
   8.409 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.410 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   queryString [null]
   8.411 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   authType    [null]
   8.412 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.413 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request headers
   8.414 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [host][localhost:8081]
   8.415 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.416 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [accept][*/*]
   8.417 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request parameters
   8.418 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] ----
   8.419 -2019-05-08 19:48:43,346 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   8.420 -2019-05-08 19:48:44,023 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   8.421 -2019-05-08 19:48:45,865 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.422 -2019-05-08 19:49:27,372 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.423 -2019-05-08 19:49:27,372 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!
   8.424 -2019-05-08 19:51:34,886 DEBUG [callback-interface-3] [AnonymousAuthenticator] Adding anonymous Authentication
   8.425 -2019-05-08 19:56:52,233 DEBUG [callback-interface-10] [HttpRequestDebug] Request properties
   8.426 -2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.427 -2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   queryString [null]
   8.428 -2019-05-08 19:56:58,423 DEBUG [callback-interface-10] [HttpRequestDebug]   authType    [null]
   8.429 -2019-05-08 19:57:02,194 DEBUG [callback-interface-10] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.430 -2019-05-08 19:57:02,936 DEBUG [callback-interface-10] [HttpRequestDebug] Request headers
   8.431 -2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [host][localhost:8081]
   8.432 -2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.433 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug]   [accept][*/*]
   8.434 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] Request parameters
   8.435 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] ----
   8.436 -2019-05-08 19:57:10,840 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Complete Status:
   8.437 -	Managed Threads: 3
   8.438 -	Active Threads: 3
   8.439 -	Active Tasks:
   8.440 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@481a442c (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0)
   8.441 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1)
   8.442 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@7998e972 (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2)
   8.443 -	Pending Tasks:
   8.444 -		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@3214a229
   8.445 -		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@2835bb63
   8.446 -Pool thread stack traces:
   8.447 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0,5,main]
   8.448 -		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   8.449 -		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   8.450 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   8.451 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.452 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1,5,main]
   8.453 -		java.util.zip.ZipFile.getEntry(ZipFile.java:314)
   8.454 -		java.util.jar.JarFile.getEntry(JarFile.java:240)
   8.455 -		java.util.jar.JarFile.getJarEntry(JarFile.java:223)
   8.456 -		org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:98)
   8.457 -		org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:256)
   8.458 -		org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281)
   8.459 -		org.apache.catalina.webresources.Cache.getResource(Cache.java:62)
   8.460 -		org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216)
   8.461 -		org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225)
   8.462 -		org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2241)
   8.463 -		org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:833)
   8.464 -		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1278)
   8.465 -		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1138)
   8.466 -		org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:250)
   8.467 -		org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
   8.468 -		org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
   8.469 -		org.postgresql.Driver.makeConnection(Driver.java:454)
   8.470 -		org.postgresql.Driver.connect(Driver.java:256)
   8.471 -		com.mchange.v2.c3p0.DriverManagerDataSource.getConnection(DriverManagerDataSource.java:134)
   8.472 -		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:182)
   8.473 -		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:171)
   8.474 -		com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool$1PooledConnectionResourcePoolManager.acquireResource(C3P0PooledConnectionPool.java:137)
   8.475 -		com.mchange.v2.resourcepool.BasicResourcePool.doAcquire(BasicResourcePool.java:1014)
   8.476 -		com.mchange.v2.resourcepool.BasicResourcePool.access$800(BasicResourcePool.java:32)
   8.477 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1810)
   8.478 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.479 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2,5,main]
   8.480 -		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   8.481 -		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   8.482 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   8.483 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.484 -
   8.485 -2019-05-08 19:59:26,545 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] Task com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (in deadlocked PoolThread) failed to complete in maximum time 60000ms. Trying interrupt().
   8.486 ---END--
   8.487 +    >   2019-05-08 19:48:30,592 DEBUG [callback-interface-1] [AnonymousAuthenticator] Adding anonymous Authentication
   8.488 +    >   2019-05-08 19:48:31,322 DEBUG [callback-interface-5] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.489 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   queryString [null]
   8.490 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   authType    [null]
   8.491 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.492 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug] Request headers
   8.493 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   [host][localhost:8081]
   8.494 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   authType    [null]
   8.495 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.496 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request headers
   8.497 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [host][localhost:8081]
   8.498 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.499 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [accept][*/*]
   8.500 +    >   2019-05-08 19:48:32,659 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.501 +    >   2019-05-08 19:48:34,236 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.502 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request parameters
   8.503 +    >   2019-05-08 19:48:34,236 DEBUG [callback-interface-5] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.504 +    >   2019-05-08 19:48:36,790 DEBUG [callback-interface-3] [HttpRequestDebug] ----
   8.505 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug]   [accept][*/*]
   8.506 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] Request parameters
   8.507 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] ----
   8.508 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug] Request properties
   8.509 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.510 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   queryString [null]
   8.511 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   authType    [null]
   8.512 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.513 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request headers
   8.514 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [host][localhost:8081]
   8.515 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.516 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [accept][*/*]
   8.517 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request parameters
   8.518 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] ----
   8.519 +    >   2019-05-08 19:48:43,346 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   8.520 +    >   2019-05-08 19:48:44,023 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   8.521 +    >   2019-05-08 19:48:45,865 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.522 +    >   2019-05-08 19:49:27,372 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   8.523 +    >   2019-05-08 19:49:27,372 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!
   8.524 +    >   2019-05-08 19:51:34,886 DEBUG [callback-interface-3] [AnonymousAuthenticator] Adding anonymous Authentication
   8.525 +    >   2019-05-08 19:56:52,233 DEBUG [callback-interface-10] [HttpRequestDebug] Request properties
   8.526 +    >   2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   8.527 +    >   2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   queryString [null]
   8.528 +    >   2019-05-08 19:56:58,423 DEBUG [callback-interface-10] [HttpRequestDebug]   authType    [null]
   8.529 +    >   2019-05-08 19:57:02,194 DEBUG [callback-interface-10] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   8.530 +    >   2019-05-08 19:57:02,936 DEBUG [callback-interface-10] [HttpRequestDebug] Request headers
   8.531 +    >   2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [host][localhost:8081]
   8.532 +    >   2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   8.533 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug]   [accept][*/*]
   8.534 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] Request parameters
   8.535 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] ----
   8.536 +    >   2019-05-08 19:57:10,840 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Complete Status:
   8.537 +    >   	Managed Threads: 3
   8.538 +    >   	Active Threads: 3
   8.539 +    >   	Active Tasks:
   8.540 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@481a442c (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0)
   8.541 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1)
   8.542 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@7998e972 (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2)
   8.543 +    >   	Pending Tasks:
   8.544 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@3214a229
   8.545 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@2835bb63
   8.546 +    >   Pool thread stack traces:
   8.547 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0,5,main]
   8.548 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   8.549 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   8.550 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   8.551 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.552 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1,5,main]
   8.553 +    >   		java.util.zip.ZipFile.getEntry(ZipFile.java:314)
   8.554 +    >   		java.util.jar.JarFile.getEntry(JarFile.java:240)
   8.555 +    >   		java.util.jar.JarFile.getJarEntry(JarFile.java:223)
   8.556 +    >   		org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:98)
   8.557 +    >   		org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:256)
   8.558 +    >   		org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281)
   8.559 +    >   		org.apache.catalina.webresources.Cache.getResource(Cache.java:62)
   8.560 +    >   		org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216)
   8.561 +    >   		org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225)
   8.562 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2241)
   8.563 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:833)
   8.564 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1278)
   8.565 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1138)
   8.566 +    >   		org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:250)
   8.567 +    >   		org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
   8.568 +    >   		org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
   8.569 +    >   		org.postgresql.Driver.makeConnection(Driver.java:454)
   8.570 +    >   		org.postgresql.Driver.connect(Driver.java:256)
   8.571 +    >   		com.mchange.v2.c3p0.DriverManagerDataSource.getConnection(DriverManagerDataSource.java:134)
   8.572 +    >   		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:182)
   8.573 +    >   		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:171)
   8.574 +    >   		com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool$1PooledConnectionResourcePoolManager.acquireResource(C3P0PooledConnectionPool.java:137)
   8.575 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.doAcquire(BasicResourcePool.java:1014)
   8.576 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$800(BasicResourcePool.java:32)
   8.577 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1810)
   8.578 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.579 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2,5,main]
   8.580 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   8.581 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   8.582 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   8.583 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   8.584 +    >   
   8.585 +    >   2019-05-08 19:59:26,545 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] Task com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (in deadlocked PoolThread) failed to complete in maximum time 60000ms. Trying interrupt().
   8.586  
   8.587      tail -n 100 logs/firethorn-trace.log
   8.588  
   8.589 ---START--
   8.590 -2019-05-08 20:17:33,686 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.591 -2019-05-08 20:17:36,580 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754910]
   8.592 -2019-05-08 20:17:36,580 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.593 -2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.594 -2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.595 -2019-05-08 20:17:40,212 TRACE [main-interface-2] [IdentityEntity] services()
   8.596 -2019-05-08 20:17:40,213 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.597 -2019-05-08 20:17:42,353 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.598 -2019-05-08 20:17:43,793 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754909]
   8.599 -2019-05-08 20:17:44,500 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.600 -2019-05-08 20:17:45,886 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.601 -2019-05-08 20:17:47,983 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.602 -2019-05-08 20:17:53,135 TRACE [main-interface-2] [IdentityEntity] services()
   8.603 -2019-05-08 20:17:54,679 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.604 -2019-05-08 20:17:57,601 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.605 -2019-05-08 20:18:01,181 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754908]
   8.606 -2019-05-08 20:18:01,897 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.607 -2019-05-08 20:18:03,326 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.608 -2019-05-08 20:18:04,806 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.609 -2019-05-08 20:18:09,791 TRACE [main-interface-2] [IdentityEntity] services()
   8.610 -2019-05-08 20:18:12,605 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.611 -2019-05-08 20:18:24,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.612 -2019-05-08 20:18:27,672 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754907]
   8.613 -2019-05-08 20:18:27,672 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.614 -2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.615 -2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.616 -2019-05-08 20:18:35,729 TRACE [main-interface-2] [IdentityEntity] services()
   8.617 -2019-05-08 20:18:38,654 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.618 -2019-05-08 20:18:40,785 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.619 -2019-05-08 20:18:44,659 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754906]
   8.620 -2019-05-08 20:18:44,659 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.621 -2019-05-08 20:18:47,137 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.622 -2019-05-08 20:18:49,387 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.623 -2019-05-08 20:18:53,563 TRACE [main-interface-2] [IdentityEntity] services()
   8.624 -2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.625 -2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.626 -2019-05-08 20:18:58,588 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754905]
   8.627 -2019-05-08 20:18:59,319 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.628 -2019-05-08 20:19:00,056 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.629 -2019-05-08 20:19:01,598 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.630 -2019-05-08 20:19:02,341 TRACE [main-interface-2] [IdentityEntity] services()
   8.631 -2019-05-08 20:19:05,328 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.632 -2019-05-08 20:19:06,127 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.633 -2019-05-08 20:19:10,942 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754904]
   8.634 -2019-05-08 20:19:14,163 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.635 -2019-05-08 20:19:17,369 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.636 -2019-05-08 20:19:26,151 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.637 -2019-05-08 20:19:30,374 TRACE [main-interface-2] [IdentityEntity] services()
   8.638 -2019-05-08 20:19:32,527 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.639 -2019-05-08 20:19:34,588 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.640 -2019-05-08 20:19:37,357 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754903]
   8.641 -2019-05-08 20:19:40,885 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.642 -2019-05-08 20:19:44,329 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.643 -2019-05-08 20:19:53,642 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.644 -2019-05-08 20:19:57,191 TRACE [main-interface-2] [IdentityEntity] services()
   8.645 -2019-05-08 20:20:00,019 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.646 -2019-05-08 20:20:10,334 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.647 -2019-05-08 20:20:20,901 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754902]
   8.648 -2019-05-08 20:20:21,729 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.649 -2019-05-08 20:20:24,094 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.650 -2019-05-08 20:20:24,873 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.651 -2019-05-08 20:20:28,789 TRACE [main-interface-2] [IdentityEntity] services()
   8.652 -2019-05-08 20:20:29,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.653 -2019-05-08 20:20:39,949 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.654 -2019-05-08 20:20:47,558 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750801]
   8.655 -2019-05-08 20:20:47,558 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.656 -2019-05-08 20:20:48,242 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.657 -2019-05-08 20:20:50,360 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.658 -2019-05-08 20:20:53,869 TRACE [main-interface-2] [IdentityEntity] services()
   8.659 -2019-05-08 20:20:56,645 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.660 -2019-05-08 20:20:59,454 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.661 -2019-05-08 20:21:04,386 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750800]
   8.662 -2019-05-08 20:21:04,386 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.663 -2019-05-08 20:21:06,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.664 -2019-05-08 20:21:09,381 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.665 -2019-05-08 20:21:12,257 TRACE [main-interface-2] [IdentityEntity] services()
   8.666 -2019-05-08 20:21:13,696 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.667 -2019-05-08 20:21:16,517 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.668 -2019-05-08 20:21:21,786 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750799]
   8.669 -2019-05-08 20:21:21,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.670 -2019-05-08 20:21:23,973 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.671 -2019-05-08 20:21:26,908 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.672 -2019-05-08 20:21:31,215 TRACE [main-interface-2] [IdentityEntity] services()
   8.673 -2019-05-08 20:21:34,123 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.674 -2019-05-08 20:21:37,228 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.675 -2019-05-08 20:21:43,020 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750798]
   8.676 -2019-05-08 20:21:45,862 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.677 -2019-05-08 20:21:47,251 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.678 -2019-05-08 20:21:50,799 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.679 -2019-05-08 20:21:54,968 TRACE [main-interface-2] [IdentityEntity] services()
   8.680 -2019-05-08 20:21:59,238 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.681 -2019-05-08 20:22:02,876 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.682 -2019-05-08 20:22:19,458 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750797]
   8.683 -2019-05-08 20:22:20,978 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.684 -2019-05-08 20:22:25,532 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.685 -2019-05-08 20:22:30,615 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.686 -2019-05-08 20:24:14,090 TRACE [main-interface-2] [AnonymousAuthenticator] afterCompletion()
   8.687 -2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] afterCompletion()
   8.688 -2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] Operation [57698710][http://gillian:8080/firethorn/adql/resource/select]
   8.689 -2019-05-08 20:24:28,569 DEBUG [main-interface-2] [OperationInterceptor] Exception [org.springframework.web.util.NestedServletException][Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space]
   8.690 ---END--
   8.691 +    >   2019-05-08 20:17:33,686 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.692 +    >   2019-05-08 20:17:36,580 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754910]
   8.693 +    >   2019-05-08 20:17:36,580 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.694 +    >   2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.695 +    >   2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.696 +    >   2019-05-08 20:17:40,212 TRACE [main-interface-2] [IdentityEntity] services()
   8.697 +    >   2019-05-08 20:17:40,213 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.698 +    >   2019-05-08 20:17:42,353 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.699 +    >   2019-05-08 20:17:43,793 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754909]
   8.700 +    >   2019-05-08 20:17:44,500 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.701 +    >   2019-05-08 20:17:45,886 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.702 +    >   2019-05-08 20:17:47,983 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.703 +    >   2019-05-08 20:17:53,135 TRACE [main-interface-2] [IdentityEntity] services()
   8.704 +    >   2019-05-08 20:17:54,679 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.705 +    >   2019-05-08 20:17:57,601 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.706 +    >   2019-05-08 20:18:01,181 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754908]
   8.707 +    >   2019-05-08 20:18:01,897 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.708 +    >   2019-05-08 20:18:03,326 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.709 +    >   2019-05-08 20:18:04,806 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.710 +    >   2019-05-08 20:18:09,791 TRACE [main-interface-2] [IdentityEntity] services()
   8.711 +    >   2019-05-08 20:18:12,605 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.712 +    >   2019-05-08 20:18:24,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.713 +    >   2019-05-08 20:18:27,672 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754907]
   8.714 +    >   2019-05-08 20:18:27,672 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.715 +    >   2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.716 +    >   2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.717 +    >   2019-05-08 20:18:35,729 TRACE [main-interface-2] [IdentityEntity] services()
   8.718 +    >   2019-05-08 20:18:38,654 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.719 +    >   2019-05-08 20:18:40,785 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.720 +    >   2019-05-08 20:18:44,659 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754906]
   8.721 +    >   2019-05-08 20:18:44,659 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.722 +    >   2019-05-08 20:18:47,137 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.723 +    >   2019-05-08 20:18:49,387 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.724 +    >   2019-05-08 20:18:53,563 TRACE [main-interface-2] [IdentityEntity] services()
   8.725 +    >   2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.726 +    >   2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.727 +    >   2019-05-08 20:18:58,588 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754905]
   8.728 +    >   2019-05-08 20:18:59,319 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.729 +    >   2019-05-08 20:19:00,056 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.730 +    >   2019-05-08 20:19:01,598 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.731 +    >   2019-05-08 20:19:02,341 TRACE [main-interface-2] [IdentityEntity] services()
   8.732 +    >   2019-05-08 20:19:05,328 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.733 +    >   2019-05-08 20:19:06,127 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.734 +    >   2019-05-08 20:19:10,942 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754904]
   8.735 +    >   2019-05-08 20:19:14,163 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.736 +    >   2019-05-08 20:19:17,369 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.737 +    >   2019-05-08 20:19:26,151 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.738 +    >   2019-05-08 20:19:30,374 TRACE [main-interface-2] [IdentityEntity] services()
   8.739 +    >   2019-05-08 20:19:32,527 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.740 +    >   2019-05-08 20:19:34,588 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.741 +    >   2019-05-08 20:19:37,357 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754903]
   8.742 +    >   2019-05-08 20:19:40,885 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.743 +    >   2019-05-08 20:19:44,329 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.744 +    >   2019-05-08 20:19:53,642 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.745 +    >   2019-05-08 20:19:57,191 TRACE [main-interface-2] [IdentityEntity] services()
   8.746 +    >   2019-05-08 20:20:00,019 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.747 +    >   2019-05-08 20:20:10,334 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.748 +    >   2019-05-08 20:20:20,901 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754902]
   8.749 +    >   2019-05-08 20:20:21,729 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.750 +    >   2019-05-08 20:20:24,094 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.751 +    >   2019-05-08 20:20:24,873 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.752 +    >   2019-05-08 20:20:28,789 TRACE [main-interface-2] [IdentityEntity] services()
   8.753 +    >   2019-05-08 20:20:29,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.754 +    >   2019-05-08 20:20:39,949 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.755 +    >   2019-05-08 20:20:47,558 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750801]
   8.756 +    >   2019-05-08 20:20:47,558 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.757 +    >   2019-05-08 20:20:48,242 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.758 +    >   2019-05-08 20:20:50,360 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.759 +    >   2019-05-08 20:20:53,869 TRACE [main-interface-2] [IdentityEntity] services()
   8.760 +    >   2019-05-08 20:20:56,645 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.761 +    >   2019-05-08 20:20:59,454 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.762 +    >   2019-05-08 20:21:04,386 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750800]
   8.763 +    >   2019-05-08 20:21:04,386 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.764 +    >   2019-05-08 20:21:06,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.765 +    >   2019-05-08 20:21:09,381 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.766 +    >   2019-05-08 20:21:12,257 TRACE [main-interface-2] [IdentityEntity] services()
   8.767 +    >   2019-05-08 20:21:13,696 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.768 +    >   2019-05-08 20:21:16,517 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.769 +    >   2019-05-08 20:21:21,786 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750799]
   8.770 +    >   2019-05-08 20:21:21,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.771 +    >   2019-05-08 20:21:23,973 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.772 +    >   2019-05-08 20:21:26,908 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.773 +    >   2019-05-08 20:21:31,215 TRACE [main-interface-2] [IdentityEntity] services()
   8.774 +    >   2019-05-08 20:21:34,123 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.775 +    >   2019-05-08 20:21:37,228 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.776 +    >   2019-05-08 20:21:43,020 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750798]
   8.777 +    >   2019-05-08 20:21:45,862 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.778 +    >   2019-05-08 20:21:47,251 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.779 +    >   2019-05-08 20:21:50,799 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.780 +    >   2019-05-08 20:21:54,968 TRACE [main-interface-2] [IdentityEntity] services()
   8.781 +    >   2019-05-08 20:21:59,238 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.782 +    >   2019-05-08 20:22:02,876 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.783 +    >   2019-05-08 20:22:19,458 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750797]
   8.784 +    >   2019-05-08 20:22:20,978 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.785 +    >   2019-05-08 20:22:25,532 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.786 +    >   2019-05-08 20:22:30,615 TRACE [main-interface-2] [AdqlResourceEntity] services()
   8.787 +    >   2019-05-08 20:24:14,090 TRACE [main-interface-2] [AnonymousAuthenticator] afterCompletion()
   8.788 +    >   2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] afterCompletion()
   8.789 +    >   2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] Operation [57698710][http://gillian:8080/firethorn/adql/resource/select]
   8.790 +    >   2019-05-08 20:24:28,569 DEBUG [main-interface-2] [OperationInterceptor] Exception [org.springframework.web.util.NestedServletException][Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space]
   8.791  
   8.792  # -----------------------------------------------------
   8.793  
   8.794 @@ -561,23 +551,21 @@
   8.795  
   8.796      df -h
   8.797  
   8.798 ---START--
   8.799 -Filesystem      Size  Used Avail Use% Mounted on
   8.800 -/dev/sda2        92G   58G   30G  66% /
   8.801 -udev             10M     0   10M   0% /dev
   8.802 -tmpfs            26G  1.2G   25G   5% /run
   8.803 -tmpfs            64G     0   64G   0% /dev/shm
   8.804 -tmpfs           5.0M     0  5.0M   0% /run/lock
   8.805 -tmpfs            64G     0   64G   0% /sys/fs/cgroup
   8.806 -/dev/sda1       511M  132K  511M   1% /boot/efi
   8.807 -/dev/sda4       9.1G   22M  8.6G   1% /tmp
   8.808 -/dev/sda5        65G  5.4G   56G   9% /var
   8.809 -/dev/sda6        53G   53M   50G   1% /home
   8.810 -/dev/sdc1       3.6T   68M  3.4T   1% /data2
   8.811 -/dev/sdb1       3.6T   74M  3.4T   1% /data1
   8.812 -tmpfs            13G     0   13G   0% /run/user/1005
   8.813 -tmpfs            13G     0   13G   0% /run/user/0
   8.814 ---END--
   8.815 +    >   Filesystem      Size  Used Avail Use% Mounted on
   8.816 +    >   /dev/sda2        92G   58G   30G  66% /
   8.817 +    >   udev             10M     0   10M   0% /dev
   8.818 +    >   tmpfs            26G  1.2G   25G   5% /run
   8.819 +    >   tmpfs            64G     0   64G   0% /dev/shm
   8.820 +    >   tmpfs           5.0M     0  5.0M   0% /run/lock
   8.821 +    >   tmpfs            64G     0   64G   0% /sys/fs/cgroup
   8.822 +    >   /dev/sda1       511M  132K  511M   1% /boot/efi
   8.823 +    >   /dev/sda4       9.1G   22M  8.6G   1% /tmp
   8.824 +    >   /dev/sda5        65G  5.4G   56G   9% /var
   8.825 +    >   /dev/sda6        53G   53M   50G   1% /home
   8.826 +    >   /dev/sdc1       3.6T   68M  3.4T   1% /data2
   8.827 +    >   /dev/sdb1       3.6T   74M  3.4T   1% /data1
   8.828 +    >   tmpfs            13G     0   13G   0% /run/user/1005
   8.829 +    >   tmpfs            13G     0   13G   0% /run/user/0
   8.830  
   8.831  # -----------------------------------------------------
   8.832  # Memory on the host is not the issue (29G cached).
   8.833 @@ -585,12 +573,10 @@
   8.834  
   8.835      free -h
   8.836  
   8.837 ---START--
   8.838 -             total       used       free     shared    buffers     cached
   8.839 -Mem:          126G        39G        86G       1.1G       297M        29G
   8.840 --/+ buffers/cache:       9.3G       116G
   8.841 -Swap:         1.9G         0B       1.9G
   8.842 ---END--
   8.843 +    >                total       used       free     shared    buffers     cached
   8.844 +    >   Mem:          126G        39G        86G       1.1G       297M        29G
   8.845 +    >   -/+ buffers/cache:       9.3G       116G
   8.846 +    >   Swap:         1.9G         0B       1.9G
   8.847  
   8.848  
   8.849  # -----------------------------------------------------
   8.850 @@ -599,17 +585,15 @@
   8.851  
   8.852      df -h
   8.853  
   8.854 ---START--
   8.855 -Filesystem      Size  Used Avail Use% Mounted on
   8.856 -devtmpfs        2.0G     0  2.0G   0% /dev
   8.857 -tmpfs           2.0G     0  2.0G   0% /dev/shm
   8.858 -tmpfs           2.0G  940K  2.0G   1% /run
   8.859 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   8.860 -/dev/vda3        31G   23G  7.7G  75% /
   8.861 -tmpfs           2.0G   25M  2.0G   2% /tmp
   8.862 -/dev/vda1       240M   89M  135M  40% /boot
   8.863 -tmpfs           395M     0  395M   0% /run/user/1001
   8.864 ---END--
   8.865 +    >   Filesystem      Size  Used Avail Use% Mounted on
   8.866 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
   8.867 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
   8.868 +    >   tmpfs           2.0G  940K  2.0G   1% /run
   8.869 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   8.870 +    >   /dev/vda3        31G   23G  7.7G  75% /
   8.871 +    >   tmpfs           2.0G   25M  2.0G   2% /tmp
   8.872 +    >   /dev/vda1       240M   89M  135M  40% /boot
   8.873 +    >   tmpfs           395M     0  395M   0% /run/user/1001
   8.874  
   8.875  
   8.876  # -----------------------------------------------------
   8.877 @@ -618,11 +602,9 @@
   8.878  
   8.879      free -h
   8.880  
   8.881 ---START--
   8.882 -              total        used        free      shared  buff/cache   available
   8.883 -Mem:           3.9G        2.1G        125M        158M        1.7G        1.4G
   8.884 -Swap:          1.0G        174M        849M
   8.885 ---END--
   8.886 +    >                 total        used        free      shared  buff/cache   available
   8.887 +    >   Mem:           3.9G        2.1G        125M        158M        1.7G        1.4G
   8.888 +    >   Swap:          1.0G        174M        849M
   8.889  
   8.890  # -----------------------------------------------------
   8.891  # Disc space in the container is not the issue (75% used).
   8.892 @@ -630,19 +612,17 @@
   8.893  
   8.894      df -h
   8.895  
   8.896 ---START--
   8.897 -Filesystem      Size  Used Avail Use% Mounted on
   8.898 -/dev/vda3        31G   23G  7.7G  75% /
   8.899 -tmpfs            64M     0   64M   0% /dev
   8.900 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   8.901 -tmpfs           2.0G     0  2.0G   0% /run
   8.902 -tmpfs           2.0G   32K  2.0G   1% /tmp
   8.903 -/dev/vda3        31G   23G  7.7G  75% /etc/firethorn.properties
   8.904 -shm              64M     0   64M   0% /dev/shm
   8.905 -tmpfs           2.0G     0  2.0G   0% /proc/acpi
   8.906 -tmpfs           2.0G     0  2.0G   0% /proc/scsi
   8.907 -tmpfs           2.0G     0  2.0G   0% /sys/firmware
   8.908 ---END--
   8.909 +    >   Filesystem      Size  Used Avail Use% Mounted on
   8.910 +    >   /dev/vda3        31G   23G  7.7G  75% /
   8.911 +    >   tmpfs            64M     0   64M   0% /dev
   8.912 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   8.913 +    >   tmpfs           2.0G     0  2.0G   0% /run
   8.914 +    >   tmpfs           2.0G   32K  2.0G   1% /tmp
   8.915 +    >   /dev/vda3        31G   23G  7.7G  75% /etc/firethorn.properties
   8.916 +    >   shm              64M     0   64M   0% /dev/shm
   8.917 +    >   tmpfs           2.0G     0  2.0G   0% /proc/acpi
   8.918 +    >   tmpfs           2.0G     0  2.0G   0% /proc/scsi
   8.919 +    >   tmpfs           2.0G     0  2.0G   0% /sys/firmware
   8.920  
   8.921  
   8.922  # -----------------------------------------------------
   8.923 @@ -656,12 +636,10 @@
   8.924              --username "${metauser:?}" \
   8.925              --dbname   "${metadata:?}"
   8.926  
   8.927 ---START--
   8.928 -psql (10.6 (Debian 10.6-1.pgdg90+1))
   8.929 -Type "help" for help.
   8.930 -
   8.931 -postgres=#
   8.932 ---END--
   8.933 +    >   psql (10.6 (Debian 10.6-1.pgdg90+1))
   8.934 +    >   Type "help" for help.
   8.935 +    >   
   8.936 +    >   postgres=#
   8.937  
   8.938  
   8.939  # -----------------------------------------------------
   8.940 @@ -670,39 +648,37 @@
   8.941  
   8.942      \dt
   8.943  
   8.944 ---START--
   8.945 -                                 List of relations
   8.946 - Schema |                   Name                    | Type  |        Owner
   8.947 ---------+-------------------------------------------+-------+----------------------
   8.948 - public | ft020116adqlcolumnentity                  | table | liengooXien1mooP0tae
   8.949 - public | ft020116adqlresourceentity                | table | liengooXien1mooP0tae
   8.950 - public | ft020116adqlschemaentity                  | table | liengooXien1mooP0tae
   8.951 - public | ft020116adqltableentity                   | table | liengooXien1mooP0tae
   8.952 - public | ft020116authmethodentity                  | table | liengooXien1mooP0tae
   8.953 - public | ft020116bluequeryentity                   | table | liengooXien1mooP0tae
   8.954 - public | ft020116bluequeryentityjointobaseresource | table | liengooXien1mooP0tae
   8.955 - public | ft020116bluetaskentity                    | table | liengooXien1mooP0tae
   8.956 - public | ft020116bluetasklogentity                 | table | liengooXien1mooP0tae
   8.957 - public | ft020116bluetaskparam                     | table | liengooXien1mooP0tae
   8.958 - public | ft020116communityentity                   | table | liengooXien1mooP0tae
   8.959 - public | ft020116configproperty                    | table | liengooXien1mooP0tae
   8.960 - public | ft020116identityentity                    | table | liengooXien1mooP0tae
   8.961 - public | ft020116ivoacolumnentity                  | table | liengooXien1mooP0tae
   8.962 - public | ft020116ivoaresourceentity                | table | liengooXien1mooP0tae
   8.963 - public | ft020116ivoaschemaentity                  | table | liengooXien1mooP0tae
   8.964 - public | ft020116ivoatableentity                   | table | liengooXien1mooP0tae
   8.965 - public | ft020116jdbccolumnentity                  | table | liengooXien1mooP0tae
   8.966 - public | ft020116jdbcresourceentity                | table | liengooXien1mooP0tae
   8.967 - public | ft020116jdbcschemaentity                  | table | liengooXien1mooP0tae
   8.968 - public | ft020116jdbctableentity                   | table | liengooXien1mooP0tae
   8.969 - public | ft020116ogsadqpresourceentity             | table | liengooXien1mooP0tae
   8.970 - public | ft020116ogsaexecresourceentity            | table | liengooXien1mooP0tae
   8.971 - public | ft020116ogsaivoaresourceentity            | table | liengooXien1mooP0tae
   8.972 - public | ft020116ogsajdbcresourceentity            | table | liengooXien1mooP0tae
   8.973 - public | ft020116ogsaserviceentity                 | table | liengooXien1mooP0tae
   8.974 - public | ft020116operationentity                   | table | liengooXien1mooP0tae
   8.975 -(27 rows)
   8.976 ---END--
   8.977 +    >                                    List of relations
   8.978 +    >    Schema |                   Name                    | Type  |        Owner
   8.979 +    >   --------+-------------------------------------------+-------+----------------------
   8.980 +    >    public | ft020116adqlcolumnentity                  | table | liengooXien1mooP0tae
   8.981 +    >    public | ft020116adqlresourceentity                | table | liengooXien1mooP0tae
   8.982 +    >    public | ft020116adqlschemaentity                  | table | liengooXien1mooP0tae
   8.983 +    >    public | ft020116adqltableentity                   | table | liengooXien1mooP0tae
   8.984 +    >    public | ft020116authmethodentity                  | table | liengooXien1mooP0tae
   8.985 +    >    public | ft020116bluequeryentity                   | table | liengooXien1mooP0tae
   8.986 +    >    public | ft020116bluequeryentityjointobaseresource | table | liengooXien1mooP0tae
   8.987 +    >    public | ft020116bluetaskentity                    | table | liengooXien1mooP0tae
   8.988 +    >    public | ft020116bluetasklogentity                 | table | liengooXien1mooP0tae
   8.989 +    >    public | ft020116bluetaskparam                     | table | liengooXien1mooP0tae
   8.990 +    >    public | ft020116communityentity                   | table | liengooXien1mooP0tae
   8.991 +    >    public | ft020116configproperty                    | table | liengooXien1mooP0tae
   8.992 +    >    public | ft020116identityentity                    | table | liengooXien1mooP0tae
   8.993 +    >    public | ft020116ivoacolumnentity                  | table | liengooXien1mooP0tae
   8.994 +    >    public | ft020116ivoaresourceentity                | table | liengooXien1mooP0tae
   8.995 +    >    public | ft020116ivoaschemaentity                  | table | liengooXien1mooP0tae
   8.996 +    >    public | ft020116ivoatableentity                   | table | liengooXien1mooP0tae
   8.997 +    >    public | ft020116jdbccolumnentity                  | table | liengooXien1mooP0tae
   8.998 +    >    public | ft020116jdbcresourceentity                | table | liengooXien1mooP0tae
   8.999 +    >    public | ft020116jdbcschemaentity                  | table | liengooXien1mooP0tae
  8.1000 +    >    public | ft020116jdbctableentity                   | table | liengooXien1mooP0tae
  8.1001 +    >    public | ft020116ogsadqpresourceentity             | table | liengooXien1mooP0tae
  8.1002 +    >    public | ft020116ogsaexecresourceentity            | table | liengooXien1mooP0tae
  8.1003 +    >    public | ft020116ogsaivoaresourceentity            | table | liengooXien1mooP0tae
  8.1004 +    >    public | ft020116ogsajdbcresourceentity            | table | liengooXien1mooP0tae
  8.1005 +    >    public | ft020116ogsaserviceentity                 | table | liengooXien1mooP0tae
  8.1006 +    >    public | ft020116operationentity                   | table | liengooXien1mooP0tae
  8.1007 +    >   (27 rows)
  8.1008  
  8.1009  # -----------------------------------------------------
  8.1010  # Check the how big the tables are.
  8.1011 @@ -710,235 +686,183 @@
  8.1012  
  8.1013      SELECT COUNT(*) FROM ft020116adqlcolumnentity ;
  8.1014  
  8.1015 ---START--
  8.1016 -  count
  8.1017 ----------
  8.1018 - 4431653
  8.1019 ---END--
  8.1020 +    >     count
  8.1021 +    >   ---------
  8.1022 +    >    4431653
  8.1023  
  8.1024      SELECT COUNT(*) FROM ft020116adqlschemaentity ;
  8.1025  
  8.1026 ---START--
  8.1027 - count
  8.1028 ---------
  8.1029 - 124803
  8.1030 -(1 row)
  8.1031 ---END--
  8.1032 +    >    count
  8.1033 +    >   --------
  8.1034 +    >    124803
  8.1035 +    >   (1 row)
  8.1036  
  8.1037      SELECT COUNT(*) FROM ft020116adqltableentity ;
  8.1038  
  8.1039 ---START--
  8.1040 - count
  8.1041 ---------
  8.1042 - 166863
  8.1043 -(1 row)
  8.1044 ---END--
  8.1045 +    >    count
  8.1046 +    >   --------
  8.1047 +    >    166863
  8.1048 +    >   (1 row)
  8.1049  
  8.1050      SELECT COUNT(*) FROM ft020116authmethodentity ;
  8.1051  
  8.1052 ---START--
  8.1053 -  count
  8.1054 -----------
  8.1055 - 16020562
  8.1056 ---END--
  8.1057 +    >     count
  8.1058 +    >   ----------
  8.1059 +    >    16020562
  8.1060  
  8.1061      SELECT COUNT(*) FROM ft020116bluequeryentity ;
  8.1062  
  8.1063 ---START--
  8.1064 - count
  8.1065 ---------
  8.1066 - 171174
  8.1067 -(1 row)
  8.1068 ---END--
  8.1069 +    >    count
  8.1070 +    >   --------
  8.1071 +    >    171174
  8.1072 +    >   (1 row)
  8.1073  
  8.1074      SELECT COUNT(*) FROM ft020116bluequeryentityjointobaseresource ;
  8.1075  
  8.1076 ---START--
  8.1077 - count
  8.1078 ---------
  8.1079 - 165455
  8.1080 -(1 row)
  8.1081 ---END--
  8.1082 +    >    count
  8.1083 +    >   --------
  8.1084 +    >    165455
  8.1085 +    >   (1 row)
  8.1086  
  8.1087      SELECT COUNT(*) FROM ft020116bluetaskentity ;
  8.1088  
  8.1089 ---START--
  8.1090 - count
  8.1091 ---------
  8.1092 - 171174
  8.1093 -(1 row)
  8.1094 ---END--
  8.1095 +    >    count
  8.1096 +    >   --------
  8.1097 +    >    171174
  8.1098 +    >   (1 row)
  8.1099  
  8.1100      SELECT COUNT(*) FROM ft020116bluetasklogentity ;
  8.1101  
  8.1102 ---START--
  8.1103 - count
  8.1104 ---------
  8.1105 - 552679
  8.1106 -(1 row)
  8.1107 ---END--
  8.1108 +    >    count
  8.1109 +    >   --------
  8.1110 +    >    552679
  8.1111 +    >   (1 row)
  8.1112  
  8.1113      SELECT COUNT(*) FROM ft020116bluetaskparam ;
  8.1114  
  8.1115 ---START--
  8.1116 - count
  8.1117 --------
  8.1118 -  2832
  8.1119 -(1 row)
  8.1120 ---END--
  8.1121 +    >    count
  8.1122 +    >   -------
  8.1123 +    >     2832
  8.1124 +    >   (1 row)
  8.1125  
  8.1126      SELECT COUNT(*) FROM ft020116communityentity ;
  8.1127  
  8.1128 ---START--
  8.1129 - count
  8.1130 --------
  8.1131 -     2
  8.1132 -(1 row)
  8.1133 ---END--
  8.1134 +    >    count
  8.1135 +    >   -------
  8.1136 +    >        2
  8.1137 +    >   (1 row)
  8.1138  
  8.1139      SELECT COUNT(*) FROM ft020116configproperty ;
  8.1140  
  8.1141 ---START--
  8.1142 - count
  8.1143 --------
  8.1144 -     0
  8.1145 -(1 row)
  8.1146 ---END--
  8.1147 +    >    count
  8.1148 +    >   -------
  8.1149 +    >        0
  8.1150 +    >   (1 row)
  8.1151  
  8.1152      SELECT COUNT(*) FROM ft020116identityentity ;
  8.1153  
  8.1154 ---START--
  8.1155 -  count
  8.1156 -----------
  8.1157 - 15313504
  8.1158 -(1 row)
  8.1159 ---END--
  8.1160 +    >     count
  8.1161 +    >   ----------
  8.1162 +    >    15313504
  8.1163 +    >   (1 row)
  8.1164  
  8.1165      SELECT COUNT(*) FROM ft020116ivoacolumnentity ;
  8.1166  
  8.1167 ---START--
  8.1168 - count
  8.1169 --------
  8.1170 -     0
  8.1171 -(1 row)
  8.1172 ---END--
  8.1173 +    >    count
  8.1174 +    >   -------
  8.1175 +    >        0
  8.1176 +    >   (1 row)
  8.1177  
  8.1178      SELECT COUNT(*) FROM ft020116ivoaresourceentity ;
  8.1179  
  8.1180 ---START--
  8.1181 - count
  8.1182 --------
  8.1183 -     0
  8.1184 -(1 row)
  8.1185 ---END--
  8.1186 +    >    count
  8.1187 +    >   -------
  8.1188 +    >        0
  8.1189 +    >   (1 row)
  8.1190  
  8.1191      SELECT COUNT(*) FROM ft020116ivoaschemaentity ;
  8.1192  
  8.1193 ---START--
  8.1194 - count
  8.1195 --------
  8.1196 -     0
  8.1197 -(1 row)
  8.1198 ---END--
  8.1199 +    >    count
  8.1200 +    >   -------
  8.1201 +    >        0
  8.1202 +    >   (1 row)
  8.1203  
  8.1204      SELECT COUNT(*) FROM ft020116ivoatableentity ;
  8.1205  
  8.1206 ---START--
  8.1207 - count
  8.1208 --------
  8.1209 -     0
  8.1210 -(1 row)
  8.1211 ---END--
  8.1212 +    >    count
  8.1213 +    >   -------
  8.1214 +    >        0
  8.1215 +    >   (1 row)
  8.1216  
  8.1217      SELECT COUNT(*) FROM ft020116jdbccolumnentity ;
  8.1218  
  8.1219 ---START--
  8.1220 -  count
  8.1221 ----------
  8.1222 - 4431979
  8.1223 -(1 row)
  8.1224 ---END--
  8.1225 +    >     count
  8.1226 +    >   ---------
  8.1227 +    >    4431979
  8.1228 +    >   (1 row)
  8.1229  
  8.1230      SELECT COUNT(*) FROM ft020116jdbcresourceentity ;
  8.1231  
  8.1232 ---START--
  8.1233 - count
  8.1234 --------
  8.1235 -    15
  8.1236 -(1 row)
  8.1237 ---END--
  8.1238 +    >    count
  8.1239 +    >   -------
  8.1240 +    >       15
  8.1241 +    >   (1 row)
  8.1242  
  8.1243      SELECT COUNT(*) FROM ft020116jdbcschemaentity ;
  8.1244  
  8.1245 ---START--
  8.1246 - count
  8.1247 --------
  8.1248 -  1206
  8.1249 -(1 row)
  8.1250 ---END--
  8.1251 +    >    count
  8.1252 +    >   -------
  8.1253 +    >     1206
  8.1254 +    >   (1 row)
  8.1255  
  8.1256      SELECT COUNT(*) FROM ft020116jdbctableentity ;
  8.1257  
  8.1258 ---START--
  8.1259 - count
  8.1260 ---------
  8.1261 - 167273
  8.1262 -(1 row)
  8.1263 ---END--
  8.1264 +    >    count
  8.1265 +    >   --------
  8.1266 +    >    167273
  8.1267 +    >   (1 row)
  8.1268  
  8.1269      SELECT COUNT(*) FROM ft020116ogsadqpresourceentity ;
  8.1270  
  8.1271 ---START--
  8.1272 - count
  8.1273 --------
  8.1274 -     0
  8.1275 -(1 row)
  8.1276 ---END--
  8.1277 +    >    count
  8.1278 +    >   -------
  8.1279 +    >        0
  8.1280 +    >   (1 row)
  8.1281  
  8.1282      SELECT COUNT(*) FROM ft020116ogsaexecresourceentity ;
  8.1283  
  8.1284 ---START--
  8.1285 - count
  8.1286 ---------
  8.1287 - 165196
  8.1288 -(1 row)
  8.1289 ---END--
  8.1290 +    >    count
  8.1291 +    >   --------
  8.1292 +    >    165196
  8.1293 +    >   (1 row)
  8.1294  
  8.1295      SELECT COUNT(*) FROM ft020116ogsaivoaresourceentity ;
  8.1296  
  8.1297 ---START--
  8.1298 - count
  8.1299 --------
  8.1300 -     0
  8.1301 -(1 row)
  8.1302 ---END--
  8.1303 +    >    count
  8.1304 +    >   -------
  8.1305 +    >        0
  8.1306 +    >   (1 row)
  8.1307  
  8.1308      SELECT COUNT(*) FROM ft020116ogsajdbcresourceentity ;
  8.1309  
  8.1310 ---START--
  8.1311 - count
  8.1312 --------
  8.1313 -     4
  8.1314 -(1 row)
  8.1315 ---END--
  8.1316 +    >    count
  8.1317 +    >   -------
  8.1318 +    >        4
  8.1319 +    >   (1 row)
  8.1320  
  8.1321      SELECT COUNT(*) FROM ft020116ogsaserviceentity ;
  8.1322  
  8.1323 ---START--
  8.1324 - count
  8.1325 --------
  8.1326 -     1
  8.1327 -(1 row)
  8.1328 ---END--
  8.1329 +    >    count
  8.1330 +    >   -------
  8.1331 +    >        1
  8.1332 +    >   (1 row)
  8.1333  
  8.1334      SELECT COUNT(*) FROM ft020116operationentity ;
  8.1335  
  8.1336 ---START--
  8.1337 -  count
  8.1338 -----------
  8.1339 - 16020567
  8.1340 -(1 row)
  8.1341 ---END--
  8.1342 +    >     count
  8.1343 +    >   ----------
  8.1344 +    >    16020567
  8.1345 +    >   (1 row)
  8.1346  
  8.1347  
  8.1348  # -----------------------------------------------------
  8.1349 @@ -1013,20 +937,18 @@
  8.1350      # authmethodentity has way too many fields.
  8.1351      # It is just a link from [identity] to [operation] with [method]
  8.1352  
  8.1353 ---START--
  8.1354 -                 Table "public.ft020116authmethodentity"
  8.1355 -  Column   |            Type             | Collation | Nullable | Default
  8.1356 ------------+-----------------------------+-----------+----------+---------
  8.1357 - ident     | bigint                      |           | not null |
  8.1358 - created   | timestamp without time zone |           | not null |
  8.1359 - modified  | timestamp without time zone |           | not null |
  8.1360 - uidhi     | bigint                      |           | not null |
  8.1361 - uidlo     | bigint                      |           | not null |
  8.1362 - method    | character varying(255)      |           |          |
  8.1363 - owner     | bigint                      |           |          |
  8.1364 - identity  | bigint                      |           | not null |
  8.1365 - operation | bigint                      |           | not null |
  8.1366 ---END--
  8.1367 +    >                    Table "public.ft020116authmethodentity"
  8.1368 +    >     Column   |            Type             | Collation | Nullable | Default
  8.1369 +    >   -----------+-----------------------------+-----------+----------+---------
  8.1370 +    >    ident     | bigint                      |           | not null |
  8.1371 +    >    created   | timestamp without time zone |           | not null |
  8.1372 +    >    modified  | timestamp without time zone |           | not null |
  8.1373 +    >    uidhi     | bigint                      |           | not null |
  8.1374 +    >    uidlo     | bigint                      |           | not null |
  8.1375 +    >    method    | character varying(255)      |           |          |
  8.1376 +    >    owner     | bigint                      |           |          |
  8.1377 +    >    identity  | bigint                      |           | not null |
  8.1378 +    >    operation | bigint                      |           | not null |
  8.1379  
  8.1380      #
  8.1381      # Options for optimisation ... but nothing that would kill the service.
  8.1382 @@ -1044,12 +966,10 @@
  8.1383              --username "${tapschemauser:?}" \
  8.1384              --dbname   "${tapschemadata:?}"
  8.1385  
  8.1386 ---START--
  8.1387 -psql (10.6 (Debian 10.6-1.pgdg90+1))
  8.1388 -Type "help" for help.
  8.1389 -
  8.1390 -postgres=#
  8.1391 ---END--
  8.1392 +    >   psql (10.6 (Debian 10.6-1.pgdg90+1))
  8.1393 +    >   Type "help" for help.
  8.1394 +    >   
  8.1395 +    >   postgres=#
  8.1396  
  8.1397  # -----------------------------------------------------
  8.1398  # List the database tables.
  8.1399 @@ -1057,9 +977,7 @@
  8.1400  
  8.1401      \dt
  8.1402  
  8.1403 ---START--
  8.1404 -Did not find any relations.
  8.1405 ---END--
  8.1406 +    >   Did not find any relations.
  8.1407  
  8.1408  
  8.1409  # -----------------------------------------------------
  8.1410 @@ -1068,27 +986,23 @@
  8.1411  
  8.1412      grep 'tapschema' chain.properties
  8.1413  
  8.1414 ---START--
  8.1415 -tapschemadata=postgres
  8.1416 -tapschemauser=ieW7oomei7Laebuu9Ax7
  8.1417 -tapschemapass=vamee3puChoomahz9Iek
  8.1418 -tapschemahost=carolina
  8.1419 -tapschemaport=5432
  8.1420 -tapschematype=pgsql
  8.1421 -tapschemajdbc=tapschemajdbc
  8.1422 ---END--
  8.1423 +    >   tapschemadata=postgres
  8.1424 +    >   tapschemauser=ieW7oomei7Laebuu9Ax7
  8.1425 +    >   tapschemapass=vamee3puChoomahz9Iek
  8.1426 +    >   tapschemahost=carolina
  8.1427 +    >   tapschemaport=5432
  8.1428 +    >   tapschematype=pgsql
  8.1429 +    >   tapschemajdbc=tapschemajdbc
  8.1430  
  8.1431  
  8.1432      grep 'tapschema' firethorn.properties
  8.1433  
  8.1434 ---START--
  8.1435 -firethorn.tapschema.resource.name=tapschemajdbc
  8.1436 -firethorn.tapschema.database.name=postgres
  8.1437 -firethorn.tapschema.database.host=carolina
  8.1438 -firethorn.tapschema.database.port=5432
  8.1439 -firethorn.tapschema.database.user=ieW7oomei7Laebuu9Ax7
  8.1440 -firethorn.tapschema.database.pass=vamee3puChoomahz9Iek
  8.1441 ---END--
  8.1442 +    >   firethorn.tapschema.resource.name=tapschemajdbc
  8.1443 +    >   firethorn.tapschema.database.name=postgres
  8.1444 +    >   firethorn.tapschema.database.host=carolina
  8.1445 +    >   firethorn.tapschema.database.port=5432
  8.1446 +    >   firethorn.tapschema.database.user=ieW7oomei7Laebuu9Ax7
  8.1447 +    >   firethorn.tapschema.database.pass=vamee3puChoomahz9Iek
  8.1448  
  8.1449  
  8.1450  # -----------------------------------------------------
  8.1451 @@ -1107,15 +1021,13 @@
  8.1452  
  8.1453      docker-compose --file "docker-compose.yml" run firethorn-py
  8.1454  
  8.1455 ---START--
  8.1456 -Starting stevedore_bethany_1  ... done
  8.1457 -Starting stevedore_jarmila_1  ... done
  8.1458 -Starting stevedore_carolina_1 ... done
  8.1459 -Starting stevedore_gillian_1  ... done
  8.1460 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
  8.1461 -[GCC 5.4.0 20160609] on linux
  8.1462 -Type "help", "copyright", "credits" or "license" for more information.
  8.1463 ---END--
  8.1464 +    >   Starting stevedore_bethany_1  ... done
  8.1465 +    >   Starting stevedore_jarmila_1  ... done
  8.1466 +    >   Starting stevedore_carolina_1 ... done
  8.1467 +    >   Starting stevedore_gillian_1  ... done
  8.1468 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
  8.1469 +    >   [GCC 5.4.0 20160609] on linux
  8.1470 +    >   Type "help", "copyright", "credits" or "license" for more information.
  8.1471  
  8.1472  
  8.1473  # -----------------------------------------------------
  8.1474 @@ -1157,23 +1069,21 @@
  8.1475      )
  8.1476  
  8.1477  print(resource)
  8.1478 ---START--
  8.1479 -{
  8.1480 -  "name": "OSA ADQL resource",
  8.1481 -  "fullname": "OSA ADQL resource",
  8.1482 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/163",
  8.1483 -  "ident": "54",
  8.1484 -  "url": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1485 -  "modified": "2018-12-12T18:44:59.556",
  8.1486 -  "schemas": "http://tap.roe.ac.uk/firethorn/adql/resource/54/schemas/select",
  8.1487 -  "queries": "http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select",
  8.1488 -  "self": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1489 -  "created": "2018-12-12T18:44:59.555",
  8.1490 -  "vosi": "http://tap.roe.ac.uk/firethorn/adql/resource/54/vosi",
  8.1491 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
  8.1492 -  "text": null
  8.1493 -}
  8.1494 ---END--
  8.1495 +    >   {
  8.1496 +    >     "name": "OSA ADQL resource",
  8.1497 +    >     "fullname": "OSA ADQL resource",
  8.1498 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/163",
  8.1499 +    >     "ident": "54",
  8.1500 +    >     "url": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1501 +    >     "modified": "2018-12-12T18:44:59.556",
  8.1502 +    >     "schemas": "http://tap.roe.ac.uk/firethorn/adql/resource/54/schemas/select",
  8.1503 +    >     "queries": "http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select",
  8.1504 +    >     "self": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1505 +    >     "created": "2018-12-12T18:44:59.555",
  8.1506 +    >     "vosi": "http://tap.roe.ac.uk/firethorn/adql/resource/54/vosi",
  8.1507 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
  8.1508 +    >     "text": null
  8.1509 +    >   }
  8.1510  
  8.1511  #
  8.1512  # Create and run a query on the resource.
  8.1513 @@ -1189,143 +1099,137 @@
  8.1514      query_obj
  8.1515      )
  8.1516  
  8.1517 ---START--
  8.1518 -{
  8.1519 -  "columns": [],
  8.1520 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1521 -  "results": {
  8.1522 -    "state": "COMPLETED",
  8.1523 -    "count": 1000,
  8.1524 -    "formats": {
  8.1525 -      "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  8.1526 -      "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  8.1527 -    },
  8.1528 -    "table": "http://tap.roe.ac.uk/firethorn/adql/table/57699203"
  8.1529 -  },
  8.1530 -  "modified": "2019-05-09T03:57:24.377",
  8.1531 -  "tables": [],
  8.1532 -  "syntax": {
  8.1533 -    "friendly": null,
  8.1534 -    "status": "VALID",
  8.1535 -    "message": null
  8.1536 -  },
  8.1537 -  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
  8.1538 -  "ident": "57699002",
  8.1539 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
  8.1540 -  "history": [
  8.1541 -    {
  8.1542 -      "state": "COMPLETED",
  8.1543 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1544 -      "modified": "2019-05-09T03:57:24.327",
  8.1545 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  8.1546 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698919",
  8.1547 -      "ident": "57699054",
  8.1548 -      "created": "2019-05-09T03:57:24.324",
  8.1549 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  8.1550 -      "level": "INFO",
  8.1551 -      "message": null,
  8.1552 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1553 -    },
  8.1554 -    {
  8.1555 -      "state": "RUNNING",
  8.1556 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1557 -      "modified": "2019-05-09T03:57:23.988",
  8.1558 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  8.1559 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698918",
  8.1560 -      "ident": "57699053",
  8.1561 -      "created": "2019-05-09T03:57:23.984",
  8.1562 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  8.1563 -      "level": "INFO",
  8.1564 -      "message": null,
  8.1565 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1566 -    },
  8.1567 -    {
  8.1568 -      "state": "READY",
  8.1569 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1570 -      "modified": "2019-05-09T03:57:20.823",
  8.1571 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  8.1572 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1573 -      "ident": "57699052",
  8.1574 -      "created": "2019-05-09T03:57:20.818",
  8.1575 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  8.1576 -      "level": "INFO",
  8.1577 -      "message": "Executing query",
  8.1578 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1579 -    }
  8.1580 -  ],
  8.1581 -  "name": "XX_5Y47JQZFGVHTOAAAAFVJVPIFRQ",
  8.1582 -  "text": null,
  8.1583 -  "fields": [],
  8.1584 -  "url": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1585 -  "limits": {
  8.1586 -    "rows": 1000000,
  8.1587 -    "time": null,
  8.1588 -    "cells": null
  8.1589 -  },
  8.1590 -  "delays": {
  8.1591 -    "every": null,
  8.1592 -    "first": null,
  8.1593 -    "last": null
  8.1594 -  },
  8.1595 -  "adql": "SELECT TOP 1000 ra , dec\nFROM ATLASDR1.atlasSource",
  8.1596 -  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1597 -  "resources": [
  8.1598 -    "http://tap.roe.ac.uk/firethorn/jdbc/resource/53"
  8.1599 -  ],
  8.1600 -  "callback": "http://tap.roe.ac.uk:8081/firethorn/callback/57699002",
  8.1601 -  "mode": "DIRECT",
  8.1602 -  "created": "2019-05-09T03:57:20.140",
  8.1603 -  "osql": "SELECT TOP 1000 [ATLASDR1].[dbo].[atlasSource].[ra] AS [ra] , [ATLASDR1].[dbo].[atlasSource].[dec] AS [dec]\nFROM [ATLASDR1].[dbo].[atlasSource]",
  8.1604 -  "status": "COMPLETED",
  8.1605 -  "self": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1606 -  "workspace": "http://tap.roe.ac.uk/firethorn/adql/resource/54"
  8.1607 -}
  8.1608 ---END--
  8.1609 +    >   {
  8.1610 +    >     "columns": [],
  8.1611 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1612 +    >     "results": {
  8.1613 +    >       "state": "COMPLETED",
  8.1614 +    >       "count": 1000,
  8.1615 +    >       "formats": {
  8.1616 +    >         "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  8.1617 +    >         "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  8.1618 +    >       },
  8.1619 +    >       "table": "http://tap.roe.ac.uk/firethorn/adql/table/57699203"
  8.1620 +    >     },
  8.1621 +    >     "modified": "2019-05-09T03:57:24.377",
  8.1622 +    >     "tables": [],
  8.1623 +    >     "syntax": {
  8.1624 +    >       "friendly": null,
  8.1625 +    >       "status": "VALID",
  8.1626 +    >       "message": null
  8.1627 +    >     },
  8.1628 +    >     "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
  8.1629 +    >     "ident": "57699002",
  8.1630 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
  8.1631 +    >     "history": [
  8.1632 +    >       {
  8.1633 +    >         "state": "COMPLETED",
  8.1634 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1635 +    >         "modified": "2019-05-09T03:57:24.327",
  8.1636 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  8.1637 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698919",
  8.1638 +    >         "ident": "57699054",
  8.1639 +    >         "created": "2019-05-09T03:57:24.324",
  8.1640 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  8.1641 +    >         "level": "INFO",
  8.1642 +    >         "message": null,
  8.1643 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1644 +    >       },
  8.1645 +    >       {
  8.1646 +    >         "state": "RUNNING",
  8.1647 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1648 +    >         "modified": "2019-05-09T03:57:23.988",
  8.1649 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  8.1650 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698918",
  8.1651 +    >         "ident": "57699053",
  8.1652 +    >         "created": "2019-05-09T03:57:23.984",
  8.1653 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  8.1654 +    >         "level": "INFO",
  8.1655 +    >         "message": null,
  8.1656 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1657 +    >       },
  8.1658 +    >       {
  8.1659 +    >         "state": "READY",
  8.1660 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1661 +    >         "modified": "2019-05-09T03:57:20.823",
  8.1662 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  8.1663 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1664 +    >         "ident": "57699052",
  8.1665 +    >         "created": "2019-05-09T03:57:20.818",
  8.1666 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  8.1667 +    >         "level": "INFO",
  8.1668 +    >         "message": "Executing query",
  8.1669 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  8.1670 +    >       }
  8.1671 +    >     ],
  8.1672 +    >     "name": "XX_5Y47JQZFGVHTOAAAAFVJVPIFRQ",
  8.1673 +    >     "text": null,
  8.1674 +    >     "fields": [],
  8.1675 +    >     "url": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1676 +    >     "limits": {
  8.1677 +    >       "rows": 1000000,
  8.1678 +    >       "time": null,
  8.1679 +    >       "cells": null
  8.1680 +    >     },
  8.1681 +    >     "delays": {
  8.1682 +    >       "every": null,
  8.1683 +    >       "first": null,
  8.1684 +    >       "last": null
  8.1685 +    >     },
  8.1686 +    >     "adql": "SELECT TOP 1000 ra , dec\nFROM ATLASDR1.atlasSource",
  8.1687 +    >     "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  8.1688 +    >     "resources": [
  8.1689 +    >       "http://tap.roe.ac.uk/firethorn/jdbc/resource/53"
  8.1690 +    >     ],
  8.1691 +    >     "callback": "http://tap.roe.ac.uk:8081/firethorn/callback/57699002",
  8.1692 +    >     "mode": "DIRECT",
  8.1693 +    >     "created": "2019-05-09T03:57:20.140",
  8.1694 +    >     "osql": "SELECT TOP 1000 [ATLASDR1].[dbo].[atlasSource].[ra] AS [ra] , [ATLASDR1].[dbo].[atlasSource].[dec] AS [dec]\nFROM [ATLASDR1].[dbo].[atlasSource]",
  8.1695 +    >     "status": "COMPLETED",
  8.1696 +    >     "self": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1697 +    >     "workspace": "http://tap.roe.ac.uk/firethorn/adql/resource/54"
  8.1698 +    >   }
  8.1699  
  8.1700  print(
  8.1701      query_obj.table()
  8.1702      )
  8.1703  
  8.1704 ---START--
  8.1705 -{
  8.1706 -  "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  8.1707 -  "fullname": "temp.XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  8.1708 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1709 -  "name": "XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  8.1710 -  "url": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  8.1711 -  "modified": "2019-05-09T03:57:20.903",
  8.1712 -  "ident": "57699203",
  8.1713 -  "depth": "PARTIAL",
  8.1714 -  "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  8.1715 -  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/57699102",
  8.1716 -  "created": "2019-05-09T03:57:20.903",
  8.1717 -  "formats": {
  8.1718 -    "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  8.1719 -    "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  8.1720 -  },
  8.1721 -  "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  8.1722 -  "columns": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/columns/select",
  8.1723 -  "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  8.1724 -  "self": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  8.1725 -  "query": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1726 -  "metadata": {
  8.1727 -    "adql": {
  8.1728 -      "count": 1000,
  8.1729 -      "status": "COMPLETED"
  8.1730 -    }
  8.1731 -  },
  8.1732 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
  8.1733 -  "text": null
  8.1734 -}
  8.1735 ---END--
  8.1736 +    >   {
  8.1737 +    >     "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  8.1738 +    >     "fullname": "temp.XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  8.1739 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  8.1740 +    >     "name": "XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  8.1741 +    >     "url": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  8.1742 +    >     "modified": "2019-05-09T03:57:20.903",
  8.1743 +    >     "ident": "57699203",
  8.1744 +    >     "depth": "PARTIAL",
  8.1745 +    >     "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  8.1746 +    >     "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/57699102",
  8.1747 +    >     "created": "2019-05-09T03:57:20.903",
  8.1748 +    >     "formats": {
  8.1749 +    >       "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  8.1750 +    >       "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  8.1751 +    >     },
  8.1752 +    >     "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  8.1753 +    >     "columns": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/columns/select",
  8.1754 +    >     "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  8.1755 +    >     "self": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  8.1756 +    >     "query": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  8.1757 +    >     "metadata": {
  8.1758 +    >       "adql": {
  8.1759 +    >         "count": 1000,
  8.1760 +    >         "status": "COMPLETED"
  8.1761 +    >       }
  8.1762 +    >     },
  8.1763 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
  8.1764 +    >     "text": null
  8.1765 +    >   }
  8.1766  
  8.1767  print(
  8.1768      query_obj.table().count()
  8.1769      )
  8.1770  
  8.1771 ---START--
  8.1772 -1000
  8.1773 ---END--
  8.1774 +    >   1000
  8.1775  
  8.1776  #
  8.1777  # Iterate the metadata tree
  8.1778 @@ -1351,50 +1255,48 @@
  8.1779          py_table = query_obj.table().as_astropy()
  8.1780          py_table.pprint()
  8.1781  
  8.1782 ---START--
  8.1783 -....
  8.1784 -....
  8.1785 -....
  8.1786 -table  [ATLASDR1][CurrentAstrometry][-1]
  8.1787 -Downloading http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable [Done]
  8.1788 -Traceback (most recent call last):
  8.1789 -  File "<stdin>", line 20, in <module>
  8.1790 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_table.py", line 97, in as_astropy
  8.1791 -    return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
  8.1792 -  File "/usr/local/lib/python3.5/dist-packages/astropy/table/table.py", line 2548, in read
  8.1793 -    out = io_registry.read(cls, *args, **kwargs)
  8.1794 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/registry.py", line 517, in read
  8.1795 -    data = reader(*args, **kwargs)
  8.1796 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/connect.py", line 73, in read_table_votable
  8.1797 -    input = parse(input, table_id=table_id)
  8.1798 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/table.py", line 137, in parse
  8.1799 -    config=config, pos=(1, 1)).parse(iterator, config)
  8.1800 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3390, in parse
  8.1801 -    iterator, tag, data, config, pos)
  8.1802 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3319, in _add_resource
  8.1803 -    resource.parse(self, iterator, config)
  8.1804 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3141, in parse
  8.1805 -    iterator, tag, data, config, pos)
  8.1806 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3092, in _add_table
  8.1807 -    table.parse(iterator, config)
  8.1808 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2382, in parse
  8.1809 -    iterator, colnumbers, fields, config)
  8.1810 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2497, in _parse_tabledata
  8.1811 -    vo_reraise(e, config, pos)
  8.1812 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  8.1813 -    raise exc
  8.1814 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2494, in _parse_tabledata
  8.1815 -    fields[i].ID))
  8.1816 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  8.1817 -    raise exc
  8.1818 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2488, in _parse_tabledata
  8.1819 -    data, config, pos)
  8.1820 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 543, in parse
  8.1821 -    value, mask = parse(x, config, pos)
  8.1822 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 815, in parse
  8.1823 -    value = int(value, 10)
  8.1824 -ValueError: http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable:1:21910: ValueError: invalid literal for int() with base 10: '0000000a' (in row 9, col 'column.57699287')
  8.1825 ---END--
  8.1826 +    >   ....
  8.1827 +    >   ....
  8.1828 +    >   ....
  8.1829 +    >   table  [ATLASDR1][CurrentAstrometry][-1]
  8.1830 +    >   Downloading http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable [Done]
  8.1831 +    >   Traceback (most recent call last):
  8.1832 +    >     File "<stdin>", line 20, in <module>
  8.1833 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_table.py", line 97, in as_astropy
  8.1834 +    >       return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
  8.1835 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/table/table.py", line 2548, in read
  8.1836 +    >       out = io_registry.read(cls, *args, **kwargs)
  8.1837 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/registry.py", line 517, in read
  8.1838 +    >       data = reader(*args, **kwargs)
  8.1839 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/connect.py", line 73, in read_table_votable
  8.1840 +    >       input = parse(input, table_id=table_id)
  8.1841 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/table.py", line 137, in parse
  8.1842 +    >       config=config, pos=(1, 1)).parse(iterator, config)
  8.1843 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3390, in parse
  8.1844 +    >       iterator, tag, data, config, pos)
  8.1845 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3319, in _add_resource
  8.1846 +    >       resource.parse(self, iterator, config)
  8.1847 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3141, in parse
  8.1848 +    >       iterator, tag, data, config, pos)
  8.1849 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3092, in _add_table
  8.1850 +    >       table.parse(iterator, config)
  8.1851 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2382, in parse
  8.1852 +    >       iterator, colnumbers, fields, config)
  8.1853 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2497, in _parse_tabledata
  8.1854 +    >       vo_reraise(e, config, pos)
  8.1855 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  8.1856 +    >       raise exc
  8.1857 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2494, in _parse_tabledata
  8.1858 +    >       fields[i].ID))
  8.1859 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  8.1860 +    >       raise exc
  8.1861 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2488, in _parse_tabledata
  8.1862 +    >       data, config, pos)
  8.1863 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 543, in parse
  8.1864 +    >       value, mask = parse(x, config, pos)
  8.1865 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 815, in parse
  8.1866 +    >       value = int(value, 10)
  8.1867 +    >   ValueError: http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable:1:21910: ValueError: invalid literal for int() with base 10: '0000000a' (in row 9, col 'column.57699287')
  8.1868  
  8.1869  
  8.1870  
  8.1871 @@ -1407,41 +1309,39 @@
  8.1872      # Requesting the large (171,174) list of queries for the main resource causes an Exception.
  8.1873      # http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select
  8.1874  
  8.1875 ---START--
  8.1876 -java.lang.OutOfMemoryError: GC overhead limit exceeded
  8.1877 -	org.hibernate.type.descriptor.sql.BigIntTypeDescriptor.getExtractor(BigIntTypeDescriptor.java:60)
  8.1878 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:261)
  8.1879 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:257)
  8.1880 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:247)
  8.1881 -	org.hibernate.type.AbstractStandardBasicType.hydrate(AbstractStandardBasicType.java:333)
  8.1882 -	org.hibernate.type.ComponentType.hydrate(ComponentType.java:663)
  8.1883 -	org.hibernate.persister.entity.AbstractEntityPersister.hydrate(AbstractEntityPersister.java:2972)
  8.1884 -	org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1746)
  8.1885 -	org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1672)
  8.1886 -	org.hibernate.loader.Loader.getRow(Loader.java:1561)
  8.1887 -	org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:731)
  8.1888 -	org.hibernate.loader.Loader.processResultSet(Loader.java:990)
  8.1889 -	org.hibernate.loader.Loader.doQuery(Loader.java:948)
  8.1890 -	org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:340)
  8.1891 -	org.hibernate.loader.Loader.doList(Loader.java:2689)
  8.1892 -	org.hibernate.loader.Loader.doList(Loader.java:2672)
  8.1893 -	org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2506)
  8.1894 -	org.hibernate.loader.Loader.list(Loader.java:2501)
  8.1895 -	org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:504)
  8.1896 -	org.hibernate.hql.internal.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:395)
  8.1897 -	org.hibernate.engine.query.spi.HQLQueryPlan.performList(HQLQueryPlan.java:221)
  8.1898 -	org.hibernate.internal.SessionImpl.list(SessionImpl.java:1508)
  8.1899 -	org.hibernate.query.internal.AbstractProducedQuery.doList(AbstractProducedQuery.java:1538)
  8.1900 -	org.hibernate.query.internal.AbstractProducedQuery.list(AbstractProducedQuery.java:1506)
  8.1901 -	uk.ac.roe.wfau.firethorn.entity.AbstractEntityFactory.list(AbstractEntityFactory.java:259)
  8.1902 -	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory.select(BlueQueryEntity.java:535)
  8.1903 -	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory$$FastClassBySpringCGLIB$$598856d8.invoke(<generated>)
  8.1904 -	org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
  8.1905 -	org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:746)
  8.1906 -	org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
  8.1907 -	org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$157/756591810.proceedWithInvocation(Unknown Source)
  8.1908 -	org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
  8.1909 ---END--
  8.1910 +    >   java.lang.OutOfMemoryError: GC overhead limit exceeded
  8.1911 +    >   	org.hibernate.type.descriptor.sql.BigIntTypeDescriptor.getExtractor(BigIntTypeDescriptor.java:60)
  8.1912 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:261)
  8.1913 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:257)
  8.1914 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:247)
  8.1915 +    >   	org.hibernate.type.AbstractStandardBasicType.hydrate(AbstractStandardBasicType.java:333)
  8.1916 +    >   	org.hibernate.type.ComponentType.hydrate(ComponentType.java:663)
  8.1917 +    >   	org.hibernate.persister.entity.AbstractEntityPersister.hydrate(AbstractEntityPersister.java:2972)
  8.1918 +    >   	org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1746)
  8.1919 +    >   	org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1672)
  8.1920 +    >   	org.hibernate.loader.Loader.getRow(Loader.java:1561)
  8.1921 +    >   	org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:731)
  8.1922 +    >   	org.hibernate.loader.Loader.processResultSet(Loader.java:990)
  8.1923 +    >   	org.hibernate.loader.Loader.doQuery(Loader.java:948)
  8.1924 +    >   	org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:340)
  8.1925 +    >   	org.hibernate.loader.Loader.doList(Loader.java:2689)
  8.1926 +    >   	org.hibernate.loader.Loader.doList(Loader.java:2672)
  8.1927 +    >   	org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2506)
  8.1928 +    >   	org.hibernate.loader.Loader.list(Loader.java:2501)
  8.1929 +    >   	org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:504)
  8.1930 +    >   	org.hibernate.hql.internal.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:395)
  8.1931 +    >   	org.hibernate.engine.query.spi.HQLQueryPlan.performList(HQLQueryPlan.java:221)
  8.1932 +    >   	org.hibernate.internal.SessionImpl.list(SessionImpl.java:1508)
  8.1933 +    >   	org.hibernate.query.internal.AbstractProducedQuery.doList(AbstractProducedQuery.java:1538)
  8.1934 +    >   	org.hibernate.query.internal.AbstractProducedQuery.list(AbstractProducedQuery.java:1506)
  8.1935 +    >   	uk.ac.roe.wfau.firethorn.entity.AbstractEntityFactory.list(AbstractEntityFactory.java:259)
  8.1936 +    >   	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory.select(BlueQueryEntity.java:535)
  8.1937 +    >   	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory$$FastClassBySpringCGLIB$$598856d8.invoke(<generated>)
  8.1938 +    >   	org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
  8.1939 +    >   	org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:746)
  8.1940 +    >   	org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
  8.1941 +    >   	org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$157/756591810.proceedWithInvocation(Unknown Source)
  8.1942 +    >   	org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
  8.1943  
  8.1944  
  8.1945      #
  8.1946 @@ -1450,9 +1350,7 @@
  8.1947  
  8.1948      \dt
  8.1949  
  8.1950 ---START--
  8.1951 -Did not find any relations.
  8.1952 ---END--
  8.1953 +    >   Did not find any relations.
  8.1954  
  8.1955      #
  8.1956      # But these resources all work.
     9.1 --- a/doc/notes/zrq/20190508-02-vmdown-debug.txt	Tue Jul 02 04:12:16 2019 +0200
     9.2 +++ b/doc/notes/zrq/20190508-02-vmdown-debug.txt	Tue Jul 02 04:12:49 2019 +0200
     9.3 @@ -35,15 +35,13 @@
     9.4      source "${HOME:?}/chain.properties"
     9.5      docker-compose --file "docker-compose.yml" run firethorn-py
     9.6  
     9.7 ---START--
     9.8 -Starting stevedore_bethany_1  ... done
     9.9 -Starting stevedore_jarmila_1  ... done
    9.10 -Starting stevedore_carolina_1 ... done
    9.11 -Starting stevedore_gillian_1  ... done
    9.12 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    9.13 -[GCC 5.4.0 20160609] on linux
    9.14 -Type "help", "copyright", "credits" or "license" for more information.
    9.15 ---END--
    9.16 +    >   Starting stevedore_bethany_1  ... done
    9.17 +    >   Starting stevedore_jarmila_1  ... done
    9.18 +    >   Starting stevedore_carolina_1 ... done
    9.19 +    >   Starting stevedore_gillian_1  ... done
    9.20 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    9.21 +    >   [GCC 5.4.0 20160609] on linux
    9.22 +    >   Type "help", "copyright", "credits" or "license" for more information.
    9.23  
    9.24  # -----------------------------------------------------
    9.25  # Check the key resources.
    9.26 @@ -88,25 +86,21 @@
    9.27  
    9.28      free -h
    9.29  
    9.30 ---START--
    9.31 -              total        used        free      shared  buff/cache   available
    9.32 -Mem:           3.9G        1.8G        245M        156M        1.8G        1.7G
    9.33 -Swap:          1.0G         71M        952M
    9.34 ---END--
    9.35 +    >                 total        used        free      shared  buff/cache   available
    9.36 +    >   Mem:           3.9G        1.8G        245M        156M        1.8G        1.7G
    9.37 +    >   Swap:          1.0G         71M        952M
    9.38  
    9.39      df -h
    9.40  
    9.41 ---START--
    9.42 -Filesystem      Size  Used Avail Use% Mounted on
    9.43 -devtmpfs        2.0G     0  2.0G   0% /dev
    9.44 -tmpfs           2.0G     0  2.0G   0% /dev/shm
    9.45 -tmpfs           2.0G  844K  2.0G   1% /run
    9.46 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
    9.47 -/dev/vda3        31G   23G  7.7G  75% /
    9.48 -tmpfs           2.0G  4.0K  2.0G   1% /tmp
    9.49 -/dev/vda1       240M   89M  135M  40% /boot
    9.50 -tmpfs           395M     0  395M   0% /run/user/1001
    9.51 ---END--
    9.52 +    >   Filesystem      Size  Used Avail Use% Mounted on
    9.53 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
    9.54 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
    9.55 +    >   tmpfs           2.0G  844K  2.0G   1% /run
    9.56 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
    9.57 +    >   /dev/vda3        31G   23G  7.7G  75% /
    9.58 +    >   tmpfs           2.0G  4.0K  2.0G   1% /tmp
    9.59 +    >   /dev/vda1       240M   89M  135M  40% /boot
    9.60 +    >   tmpfs           395M     0  395M   0% /run/user/1001
    9.61  
    9.62  
    9.63  # -----------------------------------------------------
    9.64 @@ -201,39 +195,31 @@
    9.65  
    9.66      date
    9.67  
    9.68 ---START--
    9.69 -Thu  9 May 22:45:35 BST 2019
    9.70 ---END--
    9.71 +    >   Thu  9 May 22:45:35 BST 2019
    9.72  
    9.73  free
    9.74  
    9.75 ---START--
    9.76 -              total        used        free      shared  buff/cache   available
    9.77 -Mem:        4038224     1776084      220624      159980     2041516     1871952
    9.78 -Swap:       1048572       88320      960252
    9.79 ---END--
    9.80 +    >                 total        used        free      shared  buff/cache   available
    9.81 +    >   Mem:        4038224     1776084      220624      159980     2041516     1871952
    9.82 +    >   Swap:       1048572       88320      960252
    9.83  
    9.84  free -h
    9.85  
    9.86 ---START--
    9.87 -              total        used        free      shared  buff/cache   available
    9.88 -Mem:           3.9G        1.7G        172M        156M        2.0G        1.8G
    9.89 -Swap:          1.0G         86M        937M
    9.90 ---END--
    9.91 +    >                 total        used        free      shared  buff/cache   available
    9.92 +    >   Mem:           3.9G        1.7G        172M        156M        2.0G        1.8G
    9.93 +    >   Swap:          1.0G         86M        937M
    9.94  
    9.95      df -h
    9.96  
    9.97 ---START--
    9.98 -Filesystem      Size  Used Avail Use% Mounted on
    9.99 -devtmpfs        2.0G     0  2.0G   0% /dev
   9.100 -tmpfs           2.0G     0  2.0G   0% /dev/shm
   9.101 -tmpfs           2.0G  828K  2.0G   1% /run
   9.102 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   9.103 -/dev/vda3        31G   23G  7.7G  75% /
   9.104 -tmpfs           2.0G  4.0K  2.0G   1% /tmp
   9.105 -/dev/vda1       240M   89M  135M  40% /boot
   9.106 -tmpfs           395M     0  395M   0% /run/user/1001
   9.107 ---END--
   9.108 +    >   Filesystem      Size  Used Avail Use% Mounted on
   9.109 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
   9.110 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
   9.111 +    >   tmpfs           2.0G  828K  2.0G   1% /run
   9.112 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   9.113 +    >   /dev/vda3        31G   23G  7.7G  75% /
   9.114 +    >   tmpfs           2.0G  4.0K  2.0G   1% /tmp
   9.115 +    >   /dev/vda1       240M   89M  135M  40% /boot
   9.116 +    >   tmpfs           395M     0  395M   0% /run/user/1001
   9.117  
   9.118      #
   9.119      # Memory not an issue (1.8G available).
   9.120 @@ -242,20 +228,18 @@
   9.121  
   9.122      docker ps -a
   9.123  
   9.124 ---START--
   9.125 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   9.126 -058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                18 hours ago        Exited (0) 17 hours ago                              stevedore_firethorn-py_run_7
   9.127 -e6deacef850b        firethorn/firethorn-py:2.1.28   "python3"                26 hours ago        Exited (137) 18 hours ago                            stevedore_firethorn-py_run_6
   9.128 -16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                13 days ago         Exited (137) 13 days ago                             stevedore_firethorn-py_run_5
   9.129 -2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 13 days ago                             stevedore_firethorn-py_run_4
   9.130 -659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                              stevedore_firethorn-py_run_3
   9.131 -cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                            stevedore_firethorn-py_run_2
   9.132 -ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                            stevedore_firethorn-py_run_1
   9.133 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (unhealthy)     0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   9.134 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   9.135 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_carolina_1
   9.136 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_bethany_1
   9.137 ---END--
   9.138 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   9.139 +    >   058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                18 hours ago        Exited (0) 17 hours ago                              stevedore_firethorn-py_run_7
   9.140 +    >   e6deacef850b        firethorn/firethorn-py:2.1.28   "python3"                26 hours ago        Exited (137) 18 hours ago                            stevedore_firethorn-py_run_6
   9.141 +    >   16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                13 days ago         Exited (137) 13 days ago                             stevedore_firethorn-py_run_5
   9.142 +    >   2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 13 days ago                             stevedore_firethorn-py_run_4
   9.143 +    >   659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                              stevedore_firethorn-py_run_3
   9.144 +    >   cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                            stevedore_firethorn-py_run_2
   9.145 +    >   ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                            stevedore_firethorn-py_run_1
   9.146 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (unhealthy)     0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   9.147 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   9.148 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_carolina_1
   9.149 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_bethany_1
   9.150  
   9.151      #
   9.152      # Tomcat is unhealthy because it is not responding to HTTP requests for system status.
   9.153 @@ -270,82 +254,77 @@
   9.154  
   9.155          tail -n 100 logs/firethorn-debug.log
   9.156  
   9.157 ---START--
   9.158 -2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [BaseProtector] isAdmin(Identity) [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   9.159 -2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] admin()
   9.160 -2019-05-09 21:48:00,018 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] find (Community, String) [Jo9keiRu7moo6ooNee1s]
   9.161 -2019-05-09 21:48:00,019 DEBUG [FireThornTaskScheduler-84] [IdentityEntity]   found [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   9.162 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [UserDataCleaner]   schema [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.163 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] tables() for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.164 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scan for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.165 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scantest for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.166 -....
   9.167 -....
   9.168 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] factory()
   9.169 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   9.170 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.171 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.172 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   9.173 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   9.174 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   9.175 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:50:00.017Z]
   9.176 ---END--
   9.177 +    >   2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [BaseProtector] isAdmin(Identity) [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   9.178 +    >   2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] admin()
   9.179 +    >   2019-05-09 21:48:00,018 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] find (Community, String) [Jo9keiRu7moo6ooNee1s]
   9.180 +    >   2019-05-09 21:48:00,019 DEBUG [FireThornTaskScheduler-84] [IdentityEntity]   found [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   9.181 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [UserDataCleaner]   schema [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.182 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] tables() for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.183 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scan for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.184 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scantest for [252][FirethornUserdataTAP20181212LIVE.dbo]
   9.185 +    >   ....
   9.186 +    >   ....
   9.187 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] factory()
   9.188 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   9.189 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.190 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.191 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   9.192 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   9.193 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   9.194 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:50:00.017Z]
   9.195  
   9.196  
   9.197          tail -n 200 logs/firethorn-trace.log
   9.198  
   9.199 ---START--
   9.200 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   9.201 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.202 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.203 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:28:00.081Z]
   9.204 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   9.205 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   9.206 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   9.207 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Authentications, Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   9.208 -....
   9.209 -....
   9.210 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory] scanperiod()
   9.211 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.212 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.213 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   9.214 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] scanperiod [PT25H]
   9.215 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prev scan is recent - skipping
   9.216 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:52:00.014Z]
   9.217 -2019-05-09 21:52:00,194 TRACE [FireThornTaskScheduler-93] [UserDataCleaner] -- DONE --
   9.218 -
   9.219 -    #
   9.220 -    # Need to setup a ssh tunnel for the Java debugger to see if Threads are an issue.
   9.221 -    #
   9.222 -
   9.223 -    #
   9.224 -    # Reboot the VM ..
   9.225 -    #
   9.226 -
   9.227 -# -----------------------------------------------------
   9.228 -# Increase the virtual machine memory.
   9.229 -#[user@trop02]
   9.230 -
   9.231 -    vmname=Araybwyn
   9.232 -
   9.233 -    source ${HOME}/libvirt.settings
   9.234 -
   9.235 -    virsh -c ${connection:?} \
   9.236 -        dumpxml "${vmname}"
   9.237 -
   9.238 ---START--
   9.239 -<domain type='kvm' id='9'>
   9.240 -  <name>Araybwyn</name>
   9.241 -  <uuid>a65ad4ca-0662-4c43-a32c-91e4180d87bc</uuid>
   9.242 -  <memory unit='KiB'>4194304</memory>
   9.243 -  <currentMemory unit='KiB'>4194304</currentMemory>
   9.244 -  <vcpu placement='static'>4</vcpu>
   9.245 -  <resource>
   9.246 -    <partition>/machine</partition>
   9.247 -  </resource>
   9.248 -  ....
   9.249 -</domain>
   9.250 ---END--
   9.251 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   9.252 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.253 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.254 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:28:00.081Z]
   9.255 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   9.256 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   9.257 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   9.258 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Authentications, Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   9.259 +    >   ....
   9.260 +    >   ....
   9.261 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory] scanperiod()
   9.262 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory]   value [PT25H]
   9.263 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] Factory scanperiod  [PT25H]
   9.264 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   9.265 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] scanperiod [PT25H]
   9.266 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prev scan is recent - skipping
   9.267 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:52:00.014Z]
   9.268 +    >   2019-05-09 21:52:00,194 TRACE [FireThornTaskScheduler-93] [UserDataCleaner] -- DONE --
   9.269 +    >   
   9.270 +    >       #
   9.271 +    >       # Need to setup a ssh tunnel for the Java debugger to see if Threads are an issue.
   9.272 +    >       #
   9.273 +    >   
   9.274 +    >       #
   9.275 +    >       # Reboot the VM ..
   9.276 +    >       #
   9.277 +    >   
   9.278 +    >   # -----------------------------------------------------
   9.279 +    >   # Increase the virtual machine memory.
   9.280 +    >   #[user@trop02]
   9.281 +    >   
   9.282 +    >       vmname=Araybwyn
   9.283 +    >   
   9.284 +    >       source ${HOME}/libvirt.settings
   9.285 +    >   
   9.286 +    >       virsh -c ${connection:?} \
   9.287 +    >           dumpxml "${vmname}"
   9.288 +    >   
   9.289 +    >   <domain type='kvm' id='9'>
   9.290 +    >     <name>Araybwyn</name>
   9.291 +    >     <uuid>a65ad4ca-0662-4c43-a32c-91e4180d87bc</uuid>
   9.292 +    >     <memory unit='KiB'>4194304</memory>
   9.293 +    >     <currentMemory unit='KiB'>4194304</currentMemory>
   9.294 +    >     <vcpu placement='static'>4</vcpu>
   9.295 +    >     <resource>
   9.296 +    >       <partition>/machine</partition>
   9.297 +    >     </resource>
   9.298 +    >     ....
   9.299 +    >   </domain>
   9.300  
   9.301      virsh -c ${connection:?} \
   9.302          shutdown "${vmname}"
   9.303 @@ -371,15 +350,13 @@
   9.304  
   9.305      docker ps -a
   9.306  
   9.307 ---START--
   9.308 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS               NAMES
   9.309 -058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                19 hours ago        Exited (0) 18 hours ago                          stevedore_firethorn-py_run_7
   9.310 -....
   9.311 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_gillian_1
   9.312 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_jarmila_1
   9.313 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_carolina_1
   9.314 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_bethany_1
   9.315 ---END--
   9.316 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS               NAMES
   9.317 +    >   058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                19 hours ago        Exited (0) 18 hours ago                          stevedore_firethorn-py_run_7
   9.318 +    >   ....
   9.319 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_gillian_1
   9.320 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_jarmila_1
   9.321 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_carolina_1
   9.322 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_bethany_1
   9.323  
   9.324  # -----------------------------------------------------
   9.325  # Restart our services..
   9.326 @@ -390,16 +367,14 @@
   9.327      source "${HOME:?}/chain.properties"
   9.328      docker-compose --file "docker-compose.yml" run firethorn-py
   9.329  
   9.330 ---START--
   9.331 -Starting stevedore_carolina_1 ... done
   9.332 -Starting stevedore_jarmila_1  ... done
   9.333 -Starting stevedore_bethany_1  ... done
   9.334 -Starting stevedore_gillian_1  ... done
   9.335 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
   9.336 -[GCC 5.4.0 20160609] on linux
   9.337 -Type "help", "copyright", "credits" or "license" for more information.
   9.338 ->>>
   9.339 ---END--
   9.340 +    >   Starting stevedore_carolina_1 ... done
   9.341 +    >   Starting stevedore_jarmila_1  ... done
   9.342 +    >   Starting stevedore_bethany_1  ... done
   9.343 +    >   Starting stevedore_gillian_1  ... done
   9.344 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
   9.345 +    >   [GCC 5.4.0 20160609] on linux
   9.346 +    >   Type "help", "copyright", "credits" or "license" for more information.
   9.347 +    >   >>>
   9.348  
   9.349  
   9.350  
   9.351 @@ -412,39 +387,31 @@
   9.352  
   9.353      date
   9.354  
   9.355 ---START--
   9.356 -Fri 10 May 10:59:06 BST 2019
   9.357 ---END--
   9.358 +    >   Fri 10 May 10:59:06 BST 2019
   9.359  
   9.360  free
   9.361  
   9.362 ---START--
   9.363 -              total        used        free      shared  buff/cache   available
   9.364 -Mem:        7599060     2001516     3392964      159900     2204580     5195720
   9.365 -Swap:       1048572           0     1048572
   9.366 ---END--
   9.367 +    >                 total        used        free      shared  buff/cache   available
   9.368 +    >   Mem:        7599060     2001516     3392964      159900     2204580     5195720
   9.369 +    >   Swap:       1048572           0     1048572
   9.370  
   9.371  free -h
   9.372  
   9.373 ---START--
   9.374 -              total        used        free      shared  buff/cache   available
   9.375 -Mem:           7.2G        1.9G        3.2G        156M        2.1G        5.0G
   9.376 -Swap:          1.0G          0B        1.0G
   9.377 ---END--
   9.378 +    >                 total        used        free      shared  buff/cache   available
   9.379 +    >   Mem:           7.2G        1.9G        3.2G        156M        2.1G        5.0G
   9.380 +    >   Swap:          1.0G          0B        1.0G
   9.381  
   9.382      df -h
   9.383  
   9.384 ---START--
   9.385 -Filesystem      Size  Used Avail Use% Mounted on
   9.386 -devtmpfs        3.7G     0  3.7G   0% /dev
   9.387 -tmpfs           3.7G     0  3.7G   0% /dev/shm
   9.388 -tmpfs           3.7G  780K  3.7G   1% /run
   9.389 -tmpfs           3.7G     0  3.7G   0% /sys/fs/cgroup
   9.390 -/dev/vda3        31G   23G  7.6G  75% /
   9.391 -tmpfs           3.7G  4.0K  3.7G   1% /tmp
   9.392 -/dev/vda1       240M   89M  135M  40% /boot
   9.393 -tmpfs           743M     0  743M   0% /run/user/1001
   9.394 ---END--
   9.395 +    >   Filesystem      Size  Used Avail Use% Mounted on
   9.396 +    >   devtmpfs        3.7G     0  3.7G   0% /dev
   9.397 +    >   tmpfs           3.7G     0  3.7G   0% /dev/shm
   9.398 +    >   tmpfs           3.7G  780K  3.7G   1% /run
   9.399 +    >   tmpfs           3.7G     0  3.7G   0% /sys/fs/cgroup
   9.400 +    >   /dev/vda3        31G   23G  7.6G  75% /
   9.401 +    >   tmpfs           3.7G  4.0K  3.7G   1% /tmp
   9.402 +    >   /dev/vda1       240M   89M  135M  40% /boot
   9.403 +    >   tmpfs           743M     0  743M   0% /run/user/1001
   9.404  
   9.405      #
   9.406      # Memory not an issue (5G available).
   9.407 @@ -453,14 +420,12 @@
   9.408  
   9.409      docker ps -a
   9.410  
   9.411 ---START--
   9.412 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   9.413 -....
   9.414 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   9.415 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   9.416 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_carolina_1
   9.417 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_bethany_1
   9.418 ---END--
   9.419 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   9.420 +    >   ....
   9.421 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   9.422 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   9.423 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_carolina_1
   9.424 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_bethany_1
   9.425  
   9.426  # -----------------------------------------------------
   9.427  # Check the system status.
    10.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    10.2 +++ b/doc/notes/zrq/20190601-01-hg-merge.txt	Tue Jul 02 04:12:49 2019 +0200
    10.3 @@ -0,0 +1,577 @@
    10.4 +#
    10.5 +# <meta:header>
    10.6 +#   <meta:licence>
    10.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
    10.8 +#
    10.9 +#     This information is free software: you can redistribute it and/or modify
   10.10 +#     it under the terms of the GNU General Public License as published by
   10.11 +#     the Free Software Foundation, either version 3 of the License, or
   10.12 +#     (at your option) any later version.
   10.13 +#
   10.14 +#     This information is distributed in the hope that it will be useful,
   10.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   10.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   10.17 +#     GNU General Public License for more details.
   10.18 +#
   10.19 +#     You should have received a copy of the GNU General Public License
   10.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   10.21 +#   </meta:licence>
   10.22 +# </meta:header>
   10.23 +#
   10.24 +#
   10.25 +
   10.26 +# -----------------------------------------------------
   10.27 +# Create a new VM.
   10.28 +#[user@trop01]
   10.29 +
   10.30 +    createvm
   10.31 +
   10.32 +    >   INFO : Node name [Astoalith]
   10.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
   10.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
   10.35 +    >   INFO : Disc name [Astoalith.qcow]
   10.36 +    >   INFO : Disc size [16GiB]
   10.37 +
   10.38 +    vmname=Astoalith
   10.39 +
   10.40 +
   10.41 +# -----------------------------------------------------
   10.42 +# Define a hosts lookup function.
   10.43 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
   10.44 +#[user@trop01]
   10.45 +
   10.46 +    getipv4()
   10.47 +        {
   10.48 +        getent hosts "${1:?}" | cut -d ' ' -f 1
   10.49 +        }
   10.50 +
   10.51 +
   10.52 +#---------------------------------------------------------------------
   10.53 +# Update the ssh keys for the VM.
   10.54 +#[user@trop01]
   10.55 +
   10.56 +    # Remove the current key.
   10.57 +    ssh-keygen -q -R "${vmname:?}"
   10.58 +
   10.59 +    # Add the host key(s) to known_hosts
   10.60 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
   10.61 +
   10.62 +    # Add the IP address key(s) to known_hosts
   10.63 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
   10.64 +
   10.65 +
   10.66 +# -----------------------------------------------------
   10.67 +# Login as Stevedore
   10.68 +#[user@trop]
   10.69 +
   10.70 +    ssh "${vmname:?}"
   10.71 +
   10.72 +
   10.73 +# -----------------------------------------------------
   10.74 +# Install our secret function.
   10.75 +#[user@virtual]
   10.76 +
   10.77 +    mkdir "${HOME:?}/bin"
   10.78 +    cat > "${HOME:?}/bin/secret" << 'EOF'
   10.79 +#!/bin/sh
   10.80 +ssh -n \
   10.81 +    'dave@shepseskaf.roe.ac.uk' \
   10.82 +    "bin/secret '${1}'"
   10.83 +EOF
   10.84 +
   10.85 +    chmod u+x "${HOME:?}/bin/secret"
   10.86 +    secret 'frog'
   10.87 +
   10.88 +
   10.89 +# -----------------------------------------------------
   10.90 +# Download our builder compose file
   10.91 +#[user@virtual]
   10.92 +
   10.93 +    wget -O builder.yml \
   10.94 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
   10.95 +
   10.96 +
   10.97 +# -----------------------------------------------------
   10.98 +# Set the target branch
   10.99 +#[user@virtual]
  10.100 +
  10.101 +    branch=2.1.23-zrq-update-depends
  10.102 +    branch=2.1.24-zrq-userdata-drop
  10.103 +    branch=2.1.25-zrq-tap-controller
  10.104 +    branch=2.1.31-stv-cleanup
  10.105 +    branch=2.1.32-zrq-thread-pools
  10.106 +
  10.107 +# -----------------------------------------------------
  10.108 +# Run our builder.
  10.109 +#[user@virtual]
  10.110 +
  10.111 +    export branch
  10.112 +    export secretsh=$(which secret)
  10.113 +
  10.114 +    docker-compose \
  10.115 +        --file "builder.yml" \
  10.116 +        run \
  10.117 +            builder
  10.118 +
  10.119 +    # -----------------------------------------------------
  10.120 +    # Test our secret function.
  10.121 +    #[root@builder]
  10.122 +
  10.123 +        secret 'frog'
  10.124 +
  10.125 +    # -----------------------------------------------------
  10.126 +    # Initialise our paths.
  10.127 +    #[root@builder]
  10.128 +
  10.129 +        PATH=${PATH}:/builder/bin
  10.130 +
  10.131 +        : ${FIRETHORN_HOME:=/etc/firethorn}
  10.132 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
  10.133 +
  10.134 +        export FIRETHORN_HOME
  10.135 +        export FIRETHORN_CODE
  10.136 +
  10.137 +    # -----------------------------------------------------
  10.138 +    # Checkout a copy of our source code.
  10.139 +    #[root@builder]
  10.140 +
  10.141 +        02.01-checkout.sh
  10.142 +
  10.143 +    # -----------------------------------------------------
  10.144 +    # Merge our branch into main.
  10.145 +    #[root@builder]
  10.146 +
  10.147 +        07.01-merge-start.sh
  10.148 +
  10.149 +    # -----------------------------------------------------
  10.150 +    # Update our version number.
  10.151 +    #[root@builder]
  10.152 +
  10.153 +        07.02-version-step.sh
  10.154 +
  10.155 +    # -----------------------------------------------------
  10.156 +    # Build our base images.
  10.157 +    #[root@builder]
  10.158 +
  10.159 +        04.01-buildbase.sh
  10.160 +
  10.161 +    # -----------------------------------------------------
  10.162 +    # Tag this version of builder as latest.
  10.163 +    #[root@builder]
  10.164 +
  10.165 +        source "${FIRETHORN_HOME}/merge.settings"
  10.166 +
  10.167 +        docker tag \
  10.168 +            "firethorn/builder:${newversion}" "firethorn/builder:latest"
  10.169 +
  10.170 +    # -----------------------------------------------------
  10.171 +    # Exit this version of builder.
  10.172 +    #[root@builder]
  10.173 +
  10.174 +        exit
  10.175 +
  10.176 +# -----------------------------------------------------
  10.177 +# Run a new instance of the builder.
  10.178 +#[user@virtual]
  10.179 +
  10.180 +    export branch
  10.181 +    export secretsh=$(which secret)
  10.182 +
  10.183 +    docker-compose \
  10.184 +        --file "builder.yml" \
  10.185 +        run \
  10.186 +            builder
  10.187 +
  10.188 +    # -----------------------------------------------------
  10.189 +    # Test our secret function.
  10.190 +    #[root@builder]
  10.191 +
  10.192 +        secret 'frog'
  10.193 +
  10.194 +    # -----------------------------------------------------
  10.195 +    # Initialise our paths.
  10.196 +    #[root@builder]
  10.197 +
  10.198 +        PATH=${PATH}:/builder/bin
  10.199 +
  10.200 +        : ${FIRETHORN_HOME:=/etc/firethorn}
  10.201 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
  10.202 +
  10.203 +        export FIRETHORN_HOME
  10.204 +        export FIRETHORN_CODE
  10.205 +
  10.206 +    # -----------------------------------------------------
  10.207 +    # Build our base images.
  10.208 +    #[root@builder]
  10.209 +
  10.210 +        04.01-buildbase.sh
  10.211 +
  10.212 +    # -----------------------------------------------------
  10.213 +    # Compile our Java code.
  10.214 +    #[root@builder]
  10.215 +
  10.216 +        05.01-javamaven.sh
  10.217 +
  10.218 +    # -----------------------------------------------------
  10.219 +    # Build our Java containers.
  10.220 +    #[root@builder]
  10.221 +
  10.222 +        05.02-javadocker.sh
  10.223 +
  10.224 +    # -----------------------------------------------------
  10.225 +    # Fetch our Python code.
  10.226 +    #[root@builder]
  10.227 +
  10.228 +        echo "Fetching client source"
  10.229 +
  10.230 +        ftpysrc='/var/local/build/client'
  10.231 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
  10.232 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
  10.233 +
  10.234 +        if [  -e "${ftpysrc:?}" ]
  10.235 +        then
  10.236 +            pushd "${ftpysrc:?}"
  10.237 +
  10.238 +                echo "Updating Python code"
  10.239 +                git pull
  10.240 +
  10.241 +            popd
  10.242 +        else
  10.243 +            pushd "$(dirname ${ftpysrc:?})"
  10.244 +
  10.245 +                echo "Cloning Python code from [${ftpygit:?}]"
  10.246 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
  10.247 +
  10.248 +            popd
  10.249 +        fi
  10.250 +
  10.251 +    # -----------------------------------------------------
  10.252 +    # Delete the embedded metadata.
  10.253 +    #[root@builder]
  10.254 +
  10.255 +        pushd "${ftpysrc:?}"
  10.256 +
  10.257 +            du -h  firethorn/meta
  10.258 +            rm -rf firethorn/meta
  10.259 +
  10.260 +        popd
  10.261 +
  10.262 +    # -----------------------------------------------------
  10.263 +    # Build our Python container.
  10.264 +    #[root@builder]
  10.265 +
  10.266 +        echo "Building client image"
  10.267 +
  10.268 +        pushd "${FIRETHORN_CODE:?}"
  10.269 +
  10.270 +            source 'bin/util.sh'
  10.271 +            export buildtag=$(getbuildtag)
  10.272 +            export buildsrc=$(pwd)
  10.273 +            export ftpysrc
  10.274 +
  10.275 +            docker-compose \
  10.276 +                --file docker/compose/client/firethorn-py.yml \
  10.277 +                build
  10.278 +
  10.279 +        popd
  10.280 +
  10.281 +# -----------------------------------------------------
  10.282 +# Run our Python tests, 20190601-02-merge-tests.txt
  10.283 +# -----------------------------------------------------
  10.284 +
  10.285 +    # -----------------------------------------------------
  10.286 +    # Update our Mercurial config.
  10.287 +    #[root@builder]
  10.288 +
  10.289 +        07.02-mercurial-user.sh
  10.290 +
  10.291 +    >   /var/local/build/firethorn /
  10.292 +    >   /
  10.293 +
  10.294 +
  10.295 +    # -----------------------------------------------------
  10.296 +    # Commit our merged code.
  10.297 +    #[root@builder]
  10.298 +
  10.299 +        07.03-merge-commit.sh
  10.300 +
  10.301 +    >   /var/local/build/firethorn /
  10.302 +    >   Merging [2.1.31-stv-cleanup] into [2.1.32] (Y/n)y
  10.303 +    >   Close dev branch [2.1.31-stv-cleanup] (Y/n)y
  10.304 +    >   51 files updated, 0 files merged, 0 files removed, 0 files unresolved
  10.305 +    >   51 files updated, 0 files merged, 0 files removed, 0 files unresolved
  10.306 +    >   /
  10.307 +
  10.308 +
  10.309 +    # -----------------------------------------------------
  10.310 +    # Push changes to Mercurial
  10.311 +    #[root@builder]
  10.312 +
  10.313 +        08.03-mercurial-push.sh
  10.314 +
  10.315 +    >   /var/local/build/firethorn /
  10.316 +    >   Push version [2.1.32] (Y/n)
  10.317 +    >   pushing to ssh://Zarquan@data.metagrid.co.uk//var/local/wfau/projects/firethorn/code
  10.318 +    >   The authenticity of host 'data.metagrid.co.uk (178.79.157.93)' can't be established.
  10.319 +    >   RSA key fingerprint is SHA256:iTDlBuOqyiDYLFqdkLKZH4TWz7MlavLvkP2u3yFFVwo.
  10.320 +    >   Are you sure you want to continue connecting (yes/no)? yes
  10.321 +    >   remote: Warning: Permanently added 'data.metagrid.co.uk,178.79.157.93' (RSA) to the list of known hosts.
  10.322 +    >   searching for changes
  10.323 +    >   remote: adding changesets
  10.324 +    >   remote: adding manifests
  10.325 +    >   remote: adding file changes
  10.326 +    >   remote: added 3 changesets with 52 changes to 52 files (+1 heads)
  10.327 +    >   /
  10.328 +
  10.329 +
  10.330 +    # -----------------------------------------------------
  10.331 +    # Tag our new images
  10.332 +    #[root@builder]
  10.333 +
  10.334 +        08.01-docker-latest.sh
  10.335 +
  10.336 +
  10.337 +
  10.338 +    # -----------------------------------------------------
  10.339 +    # Push images to Docker.
  10.340 +    #[root@builder]
  10.341 +
  10.342 +        08.02-docker-push.sh
  10.343 +
  10.344 +    >   WARNING! Using --password via the CLI is insecure. Use --password-stdin.
  10.345 +    >   WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
  10.346 +    >   Configure a credential helper to remove this warning. See
  10.347 +    >   https://docs.docker.com/engine/reference/commandline/login/#credentials-store
  10.348 +    >
  10.349 +    >   Login Succeeded
  10.350 +    >   The push refers to repository [docker.io/firethorn/fedora]
  10.351 +    >   8bd87b3d77b4: Pushed
  10.352 +    >   fc5a9c11c29d: Pushed
  10.353 +    >   ....
  10.354 +    >   ....
  10.355 +    >   891e1e4ef82a: Layer already exists
  10.356 +    >   2.1.32: digest: sha256:fb6357736d42559f8363223d9fe071749e3a119246da79825a08566fe84eb7ea size: 1791
  10.357 +    >   The push refers to repository [docker.io/firethorn/java]
  10.358 +    >   443d62749c73: Pushed
  10.359 +    >   8bd87b3d77b4: Mounted from firethorn/fedora
  10.360 +    >   ....
  10.361 +    >   ....
  10.362 +    >   2.1.32: digest: sha256:6c309b9af82a7152050f47a41ea7ee9e2f442fe1bf1852fedcd46039bd62bdb9 size: 2004
  10.363 +    >   The push refers to repository [docker.io/firethorn/tomcat]
  10.364 +    >   8e535c0bd17d: Pushed
  10.365 +    >   3cba4c50c98b: Pushed
  10.366 +    >   ....
  10.367 +    >   ....
  10.368 +    >   2.1.32: digest: sha256:1c3c6dcf8b9b7e5d2319f8bf7db3445e62a84224e4a37a7d30444303999e03ed size: 5754
  10.369 +    >   The push refers to repository [docker.io/firethorn/firethorn]
  10.370 +    >   a4ef9f2ca5dc: Pushed
  10.371 +    >   8bda3c0737f5: Pushed
  10.372 +    >   ....
  10.373 +    >   ....
  10.374 +    >   2.1.32: digest: sha256:d7bcce38b5170d7253adb664e34795d35e644741cfab107410228abe813532c2 size: 6174
  10.375 +    >   The push refers to repository [docker.io/firethorn/ogsadai]
  10.376 +    >   1eb420ad7c0b: Pushed
  10.377 +    >   369276ebc0e2: Pushed
  10.378 +    >   ....
  10.379 +    >   ....
  10.380 +    >   2.1.32: digest: sha256:5c090256a78574a1f75e21c245f34d8f9fb965bf1f740644188e36ba41bcd4cc size: 7210
  10.381 +    >   The push refers to repository [docker.io/firethorn/builder]
  10.382 +    >   be399e64df43: Pushed
  10.383 +    >   1daf40c55403: Pushed
  10.384 +    >   ....
  10.385 +    >   ....
  10.386 +    >   2.1.32: digest: sha256:48e0ccd96945c6a9ffc747e6849112b4fa535b1e9c3934d21daf23e75a309950 size: 6617
  10.387 +    >   The push refers to repository [docker.io/firethorn/tester]
  10.388 +    >   cd575670e127: Pushed
  10.389 +    >   2cb6a81a359e: Pushed
  10.390 +    >   ....
  10.391 +    >   ....
  10.392 +    >   2.1.32: digest: sha256:4765bb92d7eb36406b82ae590b764811de87b2c282c9e8d3c8bcf56e55a248f7 size: 2631
  10.393 +    >   The push refers to repository [docker.io/firethorn/postgres]
  10.394 +    >   9a56de1e814d: Pushed
  10.395 +    >   44f595c1dcd2: Layer already exists
  10.396 +    >   ....
  10.397 +    >   ....
  10.398 +    >   2.1.32: digest: sha256:8efb9465b6dd7a22b62da18fb3d97d73926a1ffdec4c3e9448dd39e5cfbab5c1 size: 3453
  10.399 +    >   The push refers to repository [docker.io/firethorn/sql-proxy]
  10.400 +    >   635548d2e88f: Pushed
  10.401 +    >   696c2bfb61d3: Pushed
  10.402 +    >   ....
  10.403 +    >   ....
  10.404 +    >   2.1.32: digest: sha256:4eedbc6d37e561bffcfa26f5bac99085ddd93522c8dbdc949ec8b263540c07e2 size: 3051
  10.405 +    >   The push refers to repository [docker.io/firethorn/firethorn-py]
  10.406 +    >   aaedb3375471: Pushed
  10.407 +    >   4967f15b8200: Pushed
  10.408 +    >   ....
  10.409 +    >   ....
  10.410 +    >   2.1.32: digest: sha256:4ce4b8a95c42b3ea6bfcf1a835b0c61ecc2b1a08e4b46ba80222763f85ae375f size: 3472
  10.411 +    >   The push refers to repository [docker.io/firethorn/fedora]
  10.412 +    >   8bd87b3d77b4: Layer already exists
  10.413 +    >   fc5a9c11c29d: Layer already exists
  10.414 +    >   ....
  10.415 +    >   ....
  10.416 +    >   latest: digest: sha256:fb6357736d42559f8363223d9fe071749e3a119246da79825a08566fe84eb7ea size: 1791
  10.417 +    >   The push refers to repository [docker.io/firethorn/java]
  10.418 +    >   443d62749c73: Layer already exists
  10.419 +    >   8bd87b3d77b4: Layer already exists
  10.420 +    >   ....
  10.421 +    >   ....
  10.422 +    >   latest: digest: sha256:6c309b9af82a7152050f47a41ea7ee9e2f442fe1bf1852fedcd46039bd62bdb9 size: 2004
  10.423 +    >   The push refers to repository [docker.io/firethorn/tomcat]
  10.424 +    >   8e535c0bd17d: Layer already exists
  10.425 +    >   3cba4c50c98b: Layer already exists
  10.426 +    >   ....
  10.427 +    >   ....
  10.428 +    >   latest: digest: sha256:1c3c6dcf8b9b7e5d2319f8bf7db3445e62a84224e4a37a7d30444303999e03ed size: 5754
  10.429 +    >   The push refers to repository [docker.io/firethorn/firethorn]
  10.430 +    >   a4ef9f2ca5dc: Layer already exists
  10.431 +    >   8bda3c0737f5: Layer already exists
  10.432 +    >   ....
  10.433 +    >   ....
  10.434 +    >   latest: digest: sha256:d7bcce38b5170d7253adb664e34795d35e644741cfab107410228abe813532c2 size: 6174
  10.435 +    >   The push refers to repository [docker.io/firethorn/ogsadai]
  10.436 +    >   1eb420ad7c0b: Layer already exists
  10.437 +    >   369276ebc0e2: Layer already exists
  10.438 +    >   ....
  10.439 +    >   ....
  10.440 +    >   latest: digest: sha256:5c090256a78574a1f75e21c245f34d8f9fb965bf1f740644188e36ba41bcd4cc size: 7210
  10.441 +    >   The push refers to repository [docker.io/firethorn/builder]
  10.442 +    >   be399e64df43: Layer already exists
  10.443 +    >   1daf40c55403: Layer already exists
  10.444 +    >   ....
  10.445 +    >   ....
  10.446 +    >   latest: digest: sha256:48e0ccd96945c6a9ffc747e6849112b4fa535b1e9c3934d21daf23e75a309950 size: 6617
  10.447 +    >   The push refers to repository [docker.io/firethorn/tester]
  10.448 +    >   cd575670e127: Layer already exists
  10.449 +    >   2cb6a81a359e: Layer already exists
  10.450 +    >   ....
  10.451 +    >   ....
  10.452 +    >   latest: digest: sha256:4765bb92d7eb36406b82ae590b764811de87b2c282c9e8d3c8bcf56e55a248f7 size: 2631
  10.453 +    >   The push refers to repository [docker.io/firethorn/postgres]
  10.454 +    >   9a56de1e814d: Layer already exists
  10.455 +    >   44f595c1dcd2: Layer already exists
  10.456 +    >   ....
  10.457 +    >   ....
  10.458 +    >   latest: digest: sha256:8efb9465b6dd7a22b62da18fb3d97d73926a1ffdec4c3e9448dd39e5cfbab5c1 size: 3453
  10.459 +    >   The push refers to repository [docker.io/firethorn/sql-proxy]
  10.460 +    >   635548d2e88f: Layer already exists
  10.461 +    >   696c2bfb61d3: Layer already exists
  10.462 +    >   ....
  10.463 +    >   ....
  10.464 +    >   latest: digest: sha256:4eedbc6d37e561bffcfa26f5bac99085ddd93522c8dbdc949ec8b263540c07e2 size: 3051
  10.465 +    >   The push refers to repository [docker.io/firethorn/firethorn-py]
  10.466 +    >   aaedb3375471: Layer already exists
  10.467 +    >   4967f15b8200: Layer already exists
  10.468 +    >   ....
  10.469 +    >   ....
  10.470 +    >   latest: digest: sha256:4ce4b8a95c42b3ea6bfcf1a835b0c61ecc2b1a08e4b46ba80222763f85ae375f size: 3472
  10.471 +
  10.472 +
  10.473 +    # -----------------------------------------------------
  10.474 +    # Push binaries to Maven repo.
  10.475 +    #[root@builder]
  10.476 +
  10.477 +        08.04-maven-push.sh
  10.478 +
  10.479 +    >   building file list ...
  10.480 +    >   347 files to consider
  10.481 +    >   uk/ac/roe/wfau/firethorn-admin-webapp/maven-metadata-local.xml
  10.482 +    >               318 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=340/347)
  10.483 +    >   uk/ac/roe/wfau/firethorn-admin-webapp/maven-metadata-local.xml.md5
  10.484 +    >                32 100%   31.25kB/s    0:00:00 (xfr#2, to-chk=339/347)
  10.485 +    >   ....
  10.486 +    >   ....
  10.487 +    >   Number of files: 347 (reg: 278, dir: 69)
  10.488 +    >   Number of created files: 214 (reg: 182, dir: 32)
  10.489 +    >   Number of regular files transferred: 278
  10.490 +    >   Total file size: 130.54M bytes
  10.491 +    >   Total transferred file size: 130.54M bytes
  10.492 +    >   Literal data: 130.54M bytes
  10.493 +    >   Matched data: 0 bytes
  10.494 +    >   File list size: 0
  10.495 +    >   File list generation time: 0.446 seconds
  10.496 +    >   File list transfer time: 0.000 seconds
  10.497 +    >   Total bytes sent: 130.60M
  10.498 +    >   Total bytes received: 5.97K
  10.499 +    >
  10.500 +    >   sent 130.60M bytes  received 5.97K bytes  13.75M bytes/sec
  10.501 +    >   total size is 130.54M  speedup is 1.00
  10.502 +    >
  10.503 +    >   building file list ...
  10.504 +    >   95 files to consider
  10.505 +    >   uk/org/ogsadai/ogsadai-basic-client/4.2.2198/_remote.repositories
  10.506 +    >               246 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=88/95)
  10.507 +    >   uk/org/ogsadai/ogsadai-basic-server/4.2.2198/_remote.repositories
  10.508 +    >               246 100%  240.23kB/s    0:00:00 (xfr#2, to-chk=81/95)
  10.509 +    >   ....
  10.510 +    >   ....
  10.511 +    >   Number of files: 95 (reg: 65, dir: 30)
  10.512 +    >   Number of created files: 0
  10.513 +    >   Number of regular files transferred: 13
  10.514 +    >   Total file size: 29.41M bytes
  10.515 +    >   Total transferred file size: 3.19K bytes
  10.516 +    >   Literal data: 3.19K bytes
  10.517 +    >   Matched data: 0 bytes
  10.518 +    >   File list size: 0
  10.519 +    >   File list generation time: 0.159 seconds
  10.520 +    >   File list transfer time: 0.000 seconds
  10.521 +    >   Total bytes sent: 7.33K
  10.522 +    >   Total bytes received: 336
  10.523 +    >
  10.524 +    >   sent 7.33K bytes  received 336 bytes  3.07K bytes/sec
  10.525 +    >   total size is 29.41M  speedup is 3,836.53
  10.526 +    >
  10.527 +    >   building file list ...
  10.528 +    >   31 files to consider
  10.529 +    >   uk/ac/starlink/stil-table/3.3.1/_remote.repositories
  10.530 +    >               214 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=24/31)
  10.531 +    >   uk/ac/starlink/stil-table/3.3.1/stil-table-3.3.1.jar.lastUpdated
  10.532 +    >               348 100%  339.84kB/s    0:00:00 (xfr#2, to-chk=22/31)
  10.533 +    >   ....
  10.534 +    >   ....
  10.535 +    >   Number of files: 31 (reg: 21, dir: 10)
  10.536 +    >   Number of created files: 0
  10.537 +    >   Number of regular files transferred: 9
  10.538 +    >   Total file size: 1.23M bytes
  10.539 +    >   Total transferred file size: 2.73K bytes
  10.540 +    >   Literal data: 2.73K bytes
  10.541 +    >   Matched data: 0 bytes
  10.542 +    >   File list size: 0
  10.543 +    >   File list generation time: 0.012 seconds
  10.544 +    >   File list transfer time: 0.000 seconds
  10.545 +    >   Total bytes sent: 4.14K
  10.546 +    >   Total bytes received: 236
  10.547 +    >
  10.548 +    >   sent 4.14K bytes  received 236 bytes  8.75K bytes/sec
  10.549 +    >   total size is 1.23M  speedup is 281.07
  10.550 +    >
  10.551 +    >   building file list ...
  10.552 +    >   3868 files to consider
  10.553 +    >   antlr/antlr/2.7.7/_remote.repositories
  10.554 +    >               204 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=3863/3868)
  10.555 +    >   antlr/antlr/2.7.7/antlr-2.7.7.jar.lastUpdated
  10.556 +    >               348 100%  339.84kB/s    0:00:00 (xfr#2, to-chk=3861/3868)
  10.557 +    >   ....
  10.558 +    >   ....
  10.559 +    >   ....
  10.560 +    >   ....
  10.561 +    >   Number of files: 3,868 (reg: 2,744, dir: 1,124)
  10.562 +    >   Number of created files: 0
  10.563 +    >   Number of regular files transferred: 880
  10.564 +    >   Total file size: 107.40M bytes
  10.565 +    >   Total transferred file size: 229.83K bytes
  10.566 +    >   Literal data: 229.78K bytes
  10.567 +    >   Matched data: 45 bytes
  10.568 +    >   File list size: 131.06K
  10.569 +    >   File list generation time: 1.002 seconds
  10.570 +    >   File list transfer time: 0.000 seconds
  10.571 +    >   Total bytes sent: 401.16K
  10.572 +    >   Total bytes received: 22.37K
  10.573 +    >
  10.574 +    >   sent 401.16K bytes  received 22.37K bytes  19.70K bytes/sec
  10.575 +    >   total size is 107.40M  speedup is 253.57
  10.576 +
  10.577 +
  10.578 +
  10.579 +
  10.580 +
    11.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    11.2 +++ b/doc/notes/zrq/20190601-02-merge-tests.txt	Tue Jul 02 04:12:49 2019 +0200
    11.3 @@ -0,0 +1,428 @@
    11.4 +#
    11.5 +# <meta:header>
    11.6 +#   <meta:licence>
    11.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
    11.8 +#
    11.9 +#     This information is free software: you can redistribute it and/or modify
   11.10 +#     it under the terms of the GNU General Public License as published by
   11.11 +#     the Free Software Foundation, either version 3 of the License, or
   11.12 +#     (at your option) any later version.
   11.13 +#
   11.14 +#     This information is distributed in the hope that it will be useful,
   11.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   11.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   11.17 +#     GNU General Public License for more details.
   11.18 +#
   11.19 +#     You should have received a copy of the GNU General Public License
   11.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   11.21 +#   </meta:licence>
   11.22 +# </meta:header>
   11.23 +#
   11.24 +#
   11.25 +
   11.26 +# -----------------------------------------------------
   11.27 +# Create our chain properties.
   11.28 +#[user@builder]
   11.29 +
   11.30 +    cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
   11.31 +
   11.32 +        buildtag=${buildtag:?}
   11.33 +
   11.34 +        metadata=$(pwgen 20 1)
   11.35 +        metauser=$(pwgen 20 1)
   11.36 +        metapass=$(pwgen 20 1)
   11.37 +
   11.38 +        usertype=mssql
   11.39 +        userhost=$(secret 'firethorn.user.host')
   11.40 +        userdata=$(secret 'firethorn.user.data')
   11.41 +        useruser=$(secret 'firethorn.user.user')
   11.42 +        userpass=$(secret 'firethorn.user.pass')
   11.43 +
   11.44 +        datatype=mssql
   11.45 +        datahost=$(secret 'firethorn.data.host')
   11.46 +        datadata=$(secret 'firethorn.data.data')
   11.47 +        datauser=$(secret 'firethorn.data.user')
   11.48 +        datapass=$(secret 'firethorn.data.pass')
   11.49 +
   11.50 +        tunneluser=$(secret 'ssh.tunnel.user')
   11.51 +        tunnelhost=$(secret 'ssh.tunnel.host')
   11.52 +
   11.53 +        admingroup=Hyaenidae
   11.54 +        adminuser=Aardwolf
   11.55 +        adminpass=$(pwgen 20 1)
   11.56 +
   11.57 +        guestgroup=Afrotheria
   11.58 +        guestuser=Hyrax
   11.59 +        guestpass=$(pwgen 20 1)
   11.60 +
   11.61 +        tapresource=Wilhelmina
   11.62 +        tapschemadata=data-$(pwgen 10 1)
   11.63 +        tapschemauser=user-$(pwgen 10 1)
   11.64 +        tapschemapass=pass-$(pwgen 10 1)
   11.65 +
   11.66 +EOF
   11.67 +
   11.68 +
   11.69 +# -----------------------------------------------------
   11.70 +# Determine our location.
   11.71 +#[user@builder]
   11.72 +
   11.73 +    # 'local' if VM is inside UoE
   11.74 +    # 'remote' if VM is at outside UoE
   11.75 +    # TODO Change these to 'internal' and 'external'
   11.76 +
   11.77 +    external=$(curl -4 --silent 'http://icanhazip.com/')
   11.78 +
   11.79 +    EDINBURGH='129.215.*'
   11.80 +
   11.81 +    if [[ ${external:?} == ${EDINBURGH} ]]
   11.82 +    then
   11.83 +        echo "Address [${external:?}] is in Edinburgh"
   11.84 +        #location=internal
   11.85 +        location=local
   11.86 +    else
   11.87 +        echo "Address [${external:?}] is not in Edinburgh"
   11.88 +        #location=external
   11.89 +        location=remote
   11.90 +    fi
   11.91 +
   11.92 +
   11.93 +# -----------------------------------------------------
   11.94 +# Link our compose config and start our test containers ...
   11.95 +#[user@builder]
   11.96 +
   11.97 +    # Check the env file is linked !!
   11.98 +
   11.99 +    pushd "${FIRETHORN_HOME:?}"
  11.100 +
  11.101 +        ln -sf 'chain.properties' '.env'
  11.102 +
  11.103 +        docker-compose \
  11.104 +            --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
  11.105 +            run \
  11.106 +                angela
  11.107 +
  11.108 +
  11.109 +
  11.110 +# -----------------------------------------------------
  11.111 +# -----------------------------------------------------
  11.112 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
  11.113 +#[user@virtual]
  11.114 +
  11.115 +    sudo -s
  11.116 +
  11.117 +    container=baryptera_gillian_1
  11.118 +
  11.119 +    pushd $(
  11.120 +        docker inspect \
  11.121 +            "${container:?}" \
  11.122 +      | jq -r '
  11.123 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  11.124 +            '
  11.125 +            )
  11.126 +
  11.127 +    tail -f firethorn-debug.log
  11.128 +
  11.129 +# -----------------------------------------------------
  11.130 +# -----------------------------------------------------
  11.131 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
  11.132 +#[user@virtual]
  11.133 +
  11.134 +    sudo -s
  11.135 +
  11.136 +    container=baryptera_jarmila_1
  11.137 +
  11.138 +    pushd $(
  11.139 +        docker inspect \
  11.140 +            "${container:?}" \
  11.141 +      | jq -r '
  11.142 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  11.143 +            '
  11.144 +            )
  11.145 +
  11.146 +    tail -f ogsadai.log
  11.147 +
  11.148 +# -----------------------------------------------------
  11.149 +# -----------------------------------------------------
  11.150 +# Run our Python tests ...
  11.151 +#[user@python]
  11.152 +
  11.153 +import os
  11.154 +import uuid
  11.155 +import time
  11.156 +import firethorn as ftpy
  11.157 +
  11.158 +#
  11.159 +# Create our firethorn client (using named param).
  11.160 +firethorn = ftpy.Firethorn(
  11.161 +    endpoint = os.environ.get(
  11.162 +        'endpoint'
  11.163 +        )
  11.164 +    )
  11.165 +
  11.166 +#
  11.167 +# Login as the admin account.
  11.168 +firethorn.login(
  11.169 +    os.environ.get('adminuser'),
  11.170 +    os.environ.get('adminpass'),
  11.171 +    os.environ.get('admingroup')
  11.172 +    )
  11.173 +
  11.174 +#
  11.175 +# Create a JdbcResource to connect to the ATLAS database.
  11.176 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
  11.177 +    "ATLAS JDBC resource",
  11.178 +    os.environ.get('datadata'),
  11.179 +    '*',
  11.180 +    os.environ.get('datatype'),
  11.181 +    os.environ.get('datahost'),
  11.182 +    os.environ.get('datauser'),
  11.183 +    os.environ.get('datapass')
  11.184 +    )
  11.185 +print(
  11.186 +    atlas_jdbc
  11.187 +    )
  11.188 +
  11.189 +#
  11.190 +# Create an AdqlResource to represent the JdbcResource.
  11.191 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
  11.192 +    "ATLAS ADQL resource"
  11.193 +    )
  11.194 +print(
  11.195 +    atlas_adql
  11.196 +    )
  11.197 +
  11.198 +#
  11.199 +# Import the target JdbcSchema into AdqlSchema.
  11.200 +schema_names = [
  11.201 +    "ATLASDR1"
  11.202 +    ]
  11.203 +
  11.204 +for schema_name in schema_names:
  11.205 +    print(schema_name)
  11.206 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
  11.207 +        schema_name,
  11.208 +        "dbo"
  11.209 +        )
  11.210 +    if (None != jdbc_schema):
  11.211 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
  11.212 +        adql_schema = atlas_adql.import_jdbc_schema(
  11.213 +            jdbc_schema,
  11.214 +            schema_name,
  11.215 +            metadoc=metadoc
  11.216 +            )
  11.217 +
  11.218 +#
  11.219 +# Admin user
  11.220 +# -------- -------- -------- --------
  11.221 +# Normal user
  11.222 +#
  11.223 +
  11.224 +#
  11.225 +# Login using a guest account.
  11.226 +firethorn.login(
  11.227 +    str(uuid.uuid4()),
  11.228 +    str(uuid.uuid4()),
  11.229 +    None
  11.230 +    )
  11.231 +
  11.232 +#
  11.233 +# Create a new workspace.
  11.234 +workspace = firethorn.firethorn_engine.create_adql_resource(
  11.235 +    "Query resource"
  11.236 +    )
  11.237 +
  11.238 +#
  11.239 +# Import the ATLAS schemas into our workspace
  11.240 +for schema in atlas_adql.select_schemas():
  11.241 +    workspace.import_adql_schema(
  11.242 +        schema
  11.243 +        )
  11.244 +
  11.245 +#
  11.246 +# Create and run a query.
  11.247 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  11.248 +query_obj = workspace.create_query(
  11.249 +    query_str,
  11.250 +    "COMPLETED",
  11.251 +    None,
  11.252 +    3000000
  11.253 +    )
  11.254 +print(
  11.255 +    query_obj
  11.256 +    )
  11.257 +print(
  11.258 +    query_obj.table()
  11.259 +    )
  11.260 +print(
  11.261 +    query_obj.table().count()
  11.262 +    )
  11.263 +
  11.264 +#
  11.265 +# Iterate the metadata tree
  11.266 +for schema in atlas_adql.select_schemas():
  11.267 +    for table in schema.select_tables():
  11.268 +        print(
  11.269 +            "table  [{}][{}]".format(
  11.270 +                schema.name(),
  11.271 +                table.name()
  11.272 +                )
  11.273 +            )
  11.274 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
  11.275 +            schema.name(),
  11.276 +            table.name()
  11.277 +            )
  11.278 +        query_obj = workspace.create_query(
  11.279 +            query_str,
  11.280 +            "COMPLETED",
  11.281 +            None,
  11.282 +            3000000
  11.283 +            )
  11.284 +        py_table = query_obj.table().as_astropy()
  11.285 +        py_table.pprint()
  11.286 +
  11.287 +#
  11.288 +# Run some queries in parallel
  11.289 +from concurrent.futures import ThreadPoolExecutor
  11.290 +import concurrent.futures
  11.291 +from datetime import datetime
  11.292 +
  11.293 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
  11.294 +
  11.295 +def do_query(workspace, query_str, limit, delay):
  11.296 +    before = datetime.now()
  11.297 +    query_obj = workspace.create_query(
  11.298 +        query_str,
  11.299 +        "COMPLETED",
  11.300 +        None,
  11.301 +        200000,
  11.302 +            {
  11.303 +            "adql.query.limit.rows"  : limit,
  11.304 +            "adql.query.delay.every" : delay
  11.305 +            }
  11.306 +        )
  11.307 +    after = datetime.now()
  11.308 +    return (
  11.309 +        (after - before),
  11.310 +        query_obj.json_object.get("results").get("count")
  11.311 +        )
  11.312 +
  11.313 +def do_queries(workspace, query_str, threads, delay):
  11.314 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
  11.315 +        futures = {
  11.316 +            executor.submit(
  11.317 +                do_query,
  11.318 +                workspace,
  11.319 +                query_str,
  11.320 +                limit,
  11.321 +                delay
  11.322 +                ): limit for limit in range(threads, 0, -1)
  11.323 +            }
  11.324 +        for future in concurrent.futures.as_completed(futures):
  11.325 +            print(
  11.326 +                future.result()[0],
  11.327 +                ':',
  11.328 +                future.result()[1]
  11.329 +                )
  11.330 +
  11.331 +for loop in range(1, 10):
  11.332 +    for threads in range(1, 50):
  11.333 +        for delay in range(1000, -100, -100):
  11.334 +            print("---- ", loop, threads, delay)
  11.335 +            do_queries(
  11.336 +                workspace,
  11.337 +                query_str,
  11.338 +                threads,
  11.339 +                delay
  11.340 +                )
  11.341 +
  11.342 +
  11.343 +# -------- -------- -------- --------
  11.344 +# Bug testing.
  11.345 +
  11.346 +
  11.347 +#
  11.348 +# Triggering a ChaosMonkey Exception in the cleanUp() method doesn't cause an error (the query completes).
  11.349 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l469
  11.350 +params = {}
  11.351 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity"})
  11.352 +params.update({"firethorn.monkey.data" : "baivahP0"})
  11.353 +
  11.354 +print(
  11.355 +    params
  11.356 +    )
  11.357 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  11.358 +query_obj = workspace.create_query(
  11.359 +    query_str,
  11.360 +    "COMPLETED",
  11.361 +    None,
  11.362 +    3000000,
  11.363 +    params=params
  11.364 +    )
  11.365 +print(
  11.366 +    query_obj
  11.367 +    )
  11.368 +
  11.369 +#
  11.370 +# Triggering a ChaosMonkey Exception between creating the Future and waiting for it doesn't cause the same problem (the query fails immediately).
  11.371 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l369
  11.372 +params = {}
  11.373 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity"})
  11.374 +params.update({"firethorn.monkey.data" : "uche2aNa"})
  11.375 +
  11.376 +print(
  11.377 +    params
  11.378 +    )
  11.379 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  11.380 +query_obj = workspace.create_query(
  11.381 +    query_str,
  11.382 +    "COMPLETED",
  11.383 +    None,
  11.384 +    3000000,
  11.385 +    params=params
  11.386 +    )
  11.387 +print(
  11.388 +    query_obj
  11.389 +    )
  11.390 +
  11.391 +#
  11.392 +# Trigger a ChaosMonkey Exception inside the CallableStatement replicates the same symptoms as the issue we are tracking.
  11.393 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l414
  11.394 +params = {}
  11.395 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity$ChaoticCallableStatement"})
  11.396 +params.update({"firethorn.monkey.data" : "Eoph9xie"})
  11.397 +
  11.398 +print(
  11.399 +    params
  11.400 +    )
  11.401 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  11.402 +query_obj = workspace.create_query(
  11.403 +    query_str,
  11.404 +    "COMPLETED",
  11.405 +    None,
  11.406 +    3000000,
  11.407 +    params=params
  11.408 +    )
  11.409 +print(
  11.410 +    query_obj
  11.411 +    )
  11.412 +
  11.413 +
  11.414 +
  11.415 +
  11.416 +
  11.417 +
  11.418 +#
  11.419 +# Exit the Python shell
  11.420 +
  11.421 +    Ctrl^D
  11.422 +
  11.423 +# -----------------------------------------------------
  11.424 +# Shutdown our containers ...
  11.425 +#[user@virtual]
  11.426 +
  11.427 +    docker-compose \
  11.428 +        --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-local.yml" \
  11.429 +        down
  11.430 +
  11.431 +
    12.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    12.2 +++ b/doc/notes/zrq/20190603-01-testing.txt	Tue Jul 02 04:12:49 2019 +0200
    12.3 @@ -0,0 +1,412 @@
    12.4 +#
    12.5 +# <meta:header>
    12.6 +#   <meta:licence>
    12.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
    12.8 +#
    12.9 +#     This information is free software: you can redistribute it and/or modify
   12.10 +#     it under the terms of the GNU General Public License as published by
   12.11 +#     the Free Software Foundation, either version 3 of the License, or
   12.12 +#     (at your option) any later version.
   12.13 +#
   12.14 +#     This information is distributed in the hope that it will be useful,
   12.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   12.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   12.17 +#     GNU General Public License for more details.
   12.18 +#
   12.19 +#     You should have received a copy of the GNU General Public License
   12.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   12.21 +#   </meta:licence>
   12.22 +# </meta:header>
   12.23 +#
   12.24 +#
   12.25 +
   12.26 +
   12.27 +# -----------------------------------------------------
   12.28 +# Create a new VM.
   12.29 +#[user@trop01]
   12.30 +
   12.31 +    createvm
   12.32 +
   12.33 +    >   INFO : Node name [Astoalith]
   12.34 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
   12.35 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
   12.36 +    >   INFO : Disc name [Astoalith.qcow]
   12.37 +    >   INFO : Disc size [16GiB]
   12.38 +
   12.39 +    vmname=Astoalith
   12.40 +
   12.41 +
   12.42 +# -----------------------------------------------------
   12.43 +# Define a hosts lookup function.
   12.44 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
   12.45 +#[user@trop01]
   12.46 +
   12.47 +    getipv4()
   12.48 +        {
   12.49 +        getent hosts "${1:?}" | cut -d ' ' -f 1
   12.50 +        }
   12.51 +
   12.52 +
   12.53 +#---------------------------------------------------------------------
   12.54 +# Update the ssh keys for the VM.
   12.55 +#[user@trop01]
   12.56 +
   12.57 +    # Remove the current key.
   12.58 +    ssh-keygen -q -R "${vmname:?}"
   12.59 +
   12.60 +    # Add the host key(s) to known_hosts
   12.61 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
   12.62 +
   12.63 +    # Add the IP address key(s) to known_hosts
   12.64 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
   12.65 +
   12.66 +
   12.67 +# -----------------------------------------------------
   12.68 +# Login to the virtual machine.
   12.69 +#[user@trop]
   12.70 +
   12.71 +    ssh "${vmname:?}"
   12.72 +
   12.73 +
   12.74 +# -----------------------------------------------------
   12.75 +# Install our secret function.
   12.76 +#[user@virtual]
   12.77 +
   12.78 +    mkdir "${HOME:?}/bin"
   12.79 +    cat > "${HOME:?}/bin/secret" << 'EOF'
   12.80 +#!/bin/sh
   12.81 +ssh -n \
   12.82 +    'dave@shepseskaf.roe.ac.uk' \
   12.83 +    "bin/secret '${1}'"
   12.84 +EOF
   12.85 +
   12.86 +    chmod u+x "${HOME:?}/bin/secret"
   12.87 +    secret 'frog'
   12.88 +
   12.89 +
   12.90 +# -----------------------------------------------------
   12.91 +# Create our chain properties.
   12.92 +#[user@virtual]
   12.93 +
   12.94 +    cat > "${HOME:?}/chain.properties" << EOF
   12.95 +
   12.96 +        buildtag=latest
   12.97 +
   12.98 +        metadata=$(pwgen 20 1)
   12.99 +        metauser=$(pwgen 20 1)
  12.100 +        metapass=$(pwgen 20 1)
  12.101 +
  12.102 +        usertype=mssql
  12.103 +        userhost=$(secret 'firethorn.user.host')
  12.104 +        userdata=$(secret 'firethorn.user.data')
  12.105 +        useruser=$(secret 'firethorn.user.user')
  12.106 +        userpass=$(secret 'firethorn.user.pass')
  12.107 +
  12.108 +        datatype=mssql
  12.109 +        datahost=$(secret 'firethorn.data.host')
  12.110 +        datadata=$(secret 'firethorn.data.data')
  12.111 +        datauser=$(secret 'firethorn.data.user')
  12.112 +        datapass=$(secret 'firethorn.data.pass')
  12.113 +
  12.114 +        tunneluser=$(secret 'ssh.tunnel.user')
  12.115 +        tunnelhost=$(secret 'ssh.tunnel.host')
  12.116 +
  12.117 +        admingroup=Hyaenidae
  12.118 +        adminuser=Aardwolf
  12.119 +        adminpass=$(pwgen 20 1)
  12.120 +
  12.121 +        guestgroup=Afrotheria
  12.122 +        guestuser=Hyrax
  12.123 +        guestpass=$(pwgen 20 1)
  12.124 +
  12.125 +        tapresource=Wilhelmina
  12.126 +        tapschemadata=data-$(pwgen 10 1)
  12.127 +        tapschemauser=user-$(pwgen 10 1)
  12.128 +        tapschemapass=pass-$(pwgen 10 1)
  12.129 +
  12.130 +EOF
  12.131 +
  12.132 +# -----------------------------------------------------
  12.133 +# Link our compose config.
  12.134 +#[user@virtual]
  12.135 +
  12.136 +    ln -sf "${HOME:?}/chain.properties" "${HOME:?}/.env"
  12.137 +
  12.138 +# -----------------------------------------------------
  12.139 +# Identify our location.
  12.140 +#[user@virtual]
  12.141 +
  12.142 +# Choose one.
  12.143 +# 'local' if VM is inside UoE
  12.144 +# 'remote' if VM is at outside UoE
  12.145 +# TODO Change these to 'internal' and 'external'
  12.146 +
  12.147 +    external=$(curl -4 --silent 'http://icanhazip.com/')
  12.148 +
  12.149 +    EDINBURGH='129.215.*'
  12.150 +
  12.151 +    if [[ ${external:?} == ${EDINBURGH} ]]
  12.152 +    then
  12.153 +        echo "Address [${external:?}] is in Edinburgh"
  12.154 +        #location=internal
  12.155 +        location=local
  12.156 +    else
  12.157 +        echo "Address [${external:?}] is not in Edinburgh"
  12.158 +        #location=external
  12.159 +        location=remote
  12.160 +    fi
  12.161 +
  12.162 +# -----------------------------------------------------
  12.163 +# Download our compose file.
  12.164 +#[user@virtual]
  12.165 +
  12.166 +    wget \
  12.167 +        --output-document "baryptera-${location:?}.yml" \
  12.168 +        "http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/tests/baryptera/baryptera-${location:?}.yml"
  12.169 +
  12.170 +
  12.171 +# -----------------------------------------------------
  12.172 +# Start our tests ...
  12.173 +#[user@virtual]
  12.174 +
  12.175 +    docker-compose \
  12.176 +        --file "baryptera-${location:?}.yml" \
  12.177 +        run \
  12.178 +            angela
  12.179 +
  12.180 +# -----------------------------------------------------
  12.181 +# -----------------------------------------------------
  12.182 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
  12.183 +#[user@virtual]
  12.184 +
  12.185 +    sudo -s
  12.186 +
  12.187 +    container=baryptera_gillian_1
  12.188 +    container=stevedore_gillian_1
  12.189 +
  12.190 +    pushd $(
  12.191 +        docker inspect \
  12.192 +            "${container:?}" \
  12.193 +      | jq -r '
  12.194 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  12.195 +            '
  12.196 +            )
  12.197 +
  12.198 +    tail -f firethorn-debug.log
  12.199 +
  12.200 +# -----------------------------------------------------
  12.201 +# -----------------------------------------------------
  12.202 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
  12.203 +#[user@virtual]
  12.204 +
  12.205 +    sudo -s
  12.206 +
  12.207 +    container=baryptera_jarmila_1
  12.208 +    container=stevedore_jarmila_1
  12.209 +
  12.210 +    pushd $(
  12.211 +        docker inspect \
  12.212 +            "${container:?}" \
  12.213 +      | jq -r '
  12.214 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  12.215 +            '
  12.216 +            )
  12.217 +
  12.218 +    tail -f ogsadai.log
  12.219 +
  12.220 +# -----------------------------------------------------
  12.221 +# -----------------------------------------------------
  12.222 +# Run our Python tests ...
  12.223 +#[user@python]
  12.224 +
  12.225 +import os
  12.226 +import uuid
  12.227 +import time
  12.228 +import firethorn as ftpy
  12.229 +
  12.230 +#
  12.231 +# Create our firethorn client (using named param).
  12.232 +firethorn = ftpy.Firethorn(
  12.233 +    endpoint = os.environ.get(
  12.234 +        'endpoint'
  12.235 +        )
  12.236 +    )
  12.237 +
  12.238 +#
  12.239 +# Login as the admin account.
  12.240 +firethorn.login(
  12.241 +    os.environ.get('adminuser'),
  12.242 +    os.environ.get('adminpass'),
  12.243 +    os.environ.get('admingroup')
  12.244 +    )
  12.245 +
  12.246 +#
  12.247 +# Create a JdbcResource to connect to the ATLAS database.
  12.248 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
  12.249 +    "ATLAS JDBC resource",
  12.250 +    os.environ.get('datadata'),
  12.251 +    '*',
  12.252 +    os.environ.get('datatype'),
  12.253 +    os.environ.get('datahost'),
  12.254 +    os.environ.get('datauser'),
  12.255 +    os.environ.get('datapass')
  12.256 +    )
  12.257 +print(
  12.258 +    atlas_jdbc
  12.259 +    )
  12.260 +
  12.261 +#
  12.262 +# Create an AdqlResource to represent the JdbcResource.
  12.263 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
  12.264 +    "ATLAS ADQL resource"
  12.265 +    )
  12.266 +print(
  12.267 +    atlas_adql
  12.268 +    )
  12.269 +
  12.270 +#
  12.271 +# Import the target JdbcSchema into AdqlSchema.
  12.272 +schema_names = [
  12.273 +    "ATLASDR1"
  12.274 +    ]
  12.275 +
  12.276 +for schema_name in schema_names:
  12.277 +    print(schema_name)
  12.278 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
  12.279 +        schema_name,
  12.280 +        "dbo"
  12.281 +        )
  12.282 +    if (None != jdbc_schema):
  12.283 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
  12.284 +        adql_schema = atlas_adql.import_jdbc_schema(
  12.285 +            jdbc_schema,
  12.286 +            schema_name,
  12.287 +            metadoc=metadoc
  12.288 +            )
  12.289 +
  12.290 +#
  12.291 +# Admin user
  12.292 +# -------- -------- -------- --------
  12.293 +# Normal user
  12.294 +#
  12.295 +
  12.296 +#
  12.297 +# Login using a guest account.
  12.298 +firethorn.login(
  12.299 +    str(uuid.uuid4()),
  12.300 +    str(uuid.uuid4()),
  12.301 +    None
  12.302 +    )
  12.303 +
  12.304 +#
  12.305 +# Create a new workspace.
  12.306 +workspace = firethorn.firethorn_engine.create_adql_resource(
  12.307 +    "Query resource"
  12.308 +    )
  12.309 +
  12.310 +#
  12.311 +# Import the ATLAS schemas into our workspace
  12.312 +for schema in atlas_adql.select_schemas():
  12.313 +    workspace.import_adql_schema(
  12.314 +        schema
  12.315 +        )
  12.316 +
  12.317 +#
  12.318 +# Create and run a query.
  12.319 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  12.320 +query_obj = workspace.create_query(
  12.321 +    query_str,
  12.322 +    "COMPLETED",
  12.323 +    None,
  12.324 +    3000000
  12.325 +    )
  12.326 +print(
  12.327 +    query_obj
  12.328 +    )
  12.329 +print(
  12.330 +    query_obj.table()
  12.331 +    )
  12.332 +print(
  12.333 +    query_obj.table().count()
  12.334 +    )
  12.335 +
  12.336 +#
  12.337 +# Iterate the metadata tree
  12.338 +for schema in atlas_adql.select_schemas():
  12.339 +    for table in schema.select_tables():
  12.340 +        print(
  12.341 +            "table  [{}][{}]".format(
  12.342 +                schema.name(),
  12.343 +                table.name()
  12.344 +                )
  12.345 +            )
  12.346 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
  12.347 +            schema.name(),
  12.348 +            table.name()
  12.349 +            )
  12.350 +        query_obj = workspace.create_query(
  12.351 +            query_str,
  12.352 +            "COMPLETED",
  12.353 +            None,
  12.354 +            3000000
  12.355 +            )
  12.356 +        py_table = query_obj.table().as_astropy()
  12.357 +        py_table.pprint()
  12.358 +
  12.359 +#
  12.360 +# Run multiple queries in parallel
  12.361 +from concurrent.futures import ThreadPoolExecutor
  12.362 +import concurrent.futures
  12.363 +from datetime import datetime
  12.364 +
  12.365 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
  12.366 +
  12.367 +def do_query(workspace, query_str, limit, delay):
  12.368 +    before = datetime.now()
  12.369 +    query_obj = workspace.create_query(
  12.370 +        query_str,
  12.371 +        "COMPLETED",
  12.372 +        None,
  12.373 +        200000,
  12.374 +            {
  12.375 +            "adql.query.limit.rows"  : limit,
  12.376 +            "adql.query.delay.every" : delay
  12.377 +            }
  12.378 +        )
  12.379 +    after = datetime.now()
  12.380 +    return (
  12.381 +        (after - before),
  12.382 +        query_obj.json_object.get("results").get("count")
  12.383 +        )
  12.384 +
  12.385 +def do_queries(workspace, query_str, threads, delay):
  12.386 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
  12.387 +        futures = {
  12.388 +            executor.submit(
  12.389 +                do_query,
  12.390 +                workspace,
  12.391 +                query_str,
  12.392 +                limit,
  12.393 +                delay
  12.394 +                ): limit for limit in range(threads, 0, -1)
  12.395 +            }
  12.396 +        for future in concurrent.futures.as_completed(futures):
  12.397 +            print(
  12.398 +                future.result()[0],
  12.399 +                ':',
  12.400 +                future.result()[1]
  12.401 +                )
  12.402 +
  12.403 +for loop in range(1, 10):
  12.404 +    for threads in range(1, 50):
  12.405 +        for delay in range(1000, -100, -100):
  12.406 +            print("---- ", loop, threads, delay)
  12.407 +            do_queries(
  12.408 +                workspace,
  12.409 +                query_str,
  12.410 +                threads,
  12.411 +                delay
  12.412 +                )
  12.413 +
  12.414 +
  12.415 +
    13.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    13.2 +++ b/doc/notes/zrq/20190603-02-updates.txt	Tue Jul 02 04:12:49 2019 +0200
    13.3 @@ -0,0 +1,147 @@
    13.4 +#
    13.5 +# <meta:header>
    13.6 +#   <meta:licence>
    13.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
    13.8 +#
    13.9 +#     This information is free software: you can redistribute it and/or modify
   13.10 +#     it under the terms of the GNU General Public License as published by
   13.11 +#     the Free Software Foundation, either version 3 of the License, or
   13.12 +#     (at your option) any later version.
   13.13 +#
   13.14 +#     This information is distributed in the hope that it will be useful,
   13.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   13.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   13.17 +#     GNU General Public License for more details.
   13.18 +#
   13.19 +#     You should have received a copy of the GNU General Public License
   13.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   13.21 +#   </meta:licence>
   13.22 +# </meta:header>
   13.23 +#
   13.24 +#
   13.25 +
   13.26 +    #
   13.27 +    # Dev branch to experiment with Thread pool allocations.
   13.28 +    # It is possible for the system to lock up under concurrent tests.
   13.29 +    # Incomplete callbacks from OGSA-DAI stuck waiting in the queue means everything stalls.
   13.30 +    #
   13.31 +
   13.32 +# -----------------------------------------------------
   13.33 +# Create a new development branch.
   13.34 +#[user@desktop]
   13.35 +
   13.36 +    devname=zrq-thread-pools
   13.37 +
   13.38 +    source "${HOME:?}/firethorn.settings"
   13.39 +    gedit  "${FIRETHORN_CODE:?}/doc/notes/zrq/20180302-02-hg-branch.txt" &
   13.40 +
   13.41 +
   13.42 +#---------------------------------------------------------------------
   13.43 +# Update our base container.
   13.44 +#[user@desktop]
   13.45 +
   13.46 +    source "${HOME:?}/firethorn.settings"
   13.47 +    pushd "${FIRETHORN_CODE:?}"
   13.48 +
   13.49 +        vi docker/fedora/Dockerfile
   13.50 +
   13.51 +        -   # FROM fedora:28
   13.52 +        -   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
   13.53 +        -   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   13.54 +
   13.55 +        +   # FROM latest (fedora:29)
   13.56 +        +   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
   13.57 +        +   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   13.58 +
   13.59 +
   13.60 +        vi docker/java/Dockerfile
   13.61 +
   13.62 +        ~   ENV JAVA_VERSION=12.0.0.33
   13.63 +
   13.64 +
   13.65 +        vi docker/tomcat/Dockerfile
   13.66 +
   13.67 +        -   ENV TOMCAT_VERSION 9.0.10
   13.68 +        +   ENV TOMCAT_VERSION 9.0.20
   13.69 +
   13.70 +
   13.71 +        vi docker/tomcat/Dockerfile
   13.72 +
   13.73 +        -   FROM postgres:10
   13.74 +        +   FROM postgres:11
   13.75 +
   13.76 +
   13.77 +    popd
   13.78 +
   13.79 +
   13.80 +# -----------------------------------------------------
   13.81 +# Build our Docker images.
   13.82 +#[user@desktop]
   13.83 +
   13.84 +    source "${HOME:?}/firethorn.settings"
   13.85 +    pushd "${FIRETHORN_CODE:?}"
   13.86 +
   13.87 +        export buildtag=$(hg branch)
   13.88 +
   13.89 +        docker-compose \
   13.90 +            --file docker/compose/images.yml \
   13.91 +            build
   13.92 +
   13.93 +    popd
   13.94 +
   13.95 +
   13.96 +#---------------------------------------------------------------------
   13.97 +# Update our Java dependencies.
   13.98 +#[user@desktop]
   13.99 +
  13.100 +    source "${HOME:?}/firethorn.settings"
  13.101 +    pushd "${FIRETHORN_CODE:?}"
  13.102 +
  13.103 +        mvn versions:display-dependency-updates
  13.104 +
  13.105 +    popd
  13.106 +
  13.107 +
  13.108 +#---------------------------------------------------------------------
  13.109 +# Update our Maven dependencies.
  13.110 +#[user@desktop]
  13.111 +
  13.112 +    source "${HOME:?}/firethorn.settings"
  13.113 +    pushd "${FIRETHORN_CODE:?}"
  13.114 +
  13.115 +        mvn versions:display-plugin-updates
  13.116 +
  13.117 +    popd
  13.118 +
  13.119 +
  13.120 +#---------------------------------------------------------------------
  13.121 +# Compile our Java code.
  13.122 +#[user@desktop]
  13.123 +
  13.124 +    source "${HOME:?}/firethorn.settings"
  13.125 +    pushd "${FIRETHORN_CODE:?}"
  13.126 +
  13.127 +        mvn clean install eclipse:eclipse
  13.128 +
  13.129 +    popd
  13.130 +
  13.131 +# -----------------------------------------------------
  13.132 +# Build our Java containers.
  13.133 +#[user@desktop]
  13.134 +
  13.135 +    source "${HOME:?}/firethorn.settings"
  13.136 +    pushd "${FIRETHORN_CODE:?}"
  13.137 +
  13.138 +        export buildtag=$(hg branch)
  13.139 +        pushd firethorn-ogsadai/webapp
  13.140 +            mvn docker:package
  13.141 +        popd
  13.142 +
  13.143 +        export buildtag=$(hg branch)
  13.144 +        pushd firethorn-webapp
  13.145 +            mvn docker:package
  13.146 +        popd
  13.147 +
  13.148 +    popd
  13.149 +
  13.150 +
    14.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    14.2 +++ b/doc/notes/zrq/20190604-01-test-build.txt	Tue Jul 02 04:12:49 2019 +0200
    14.3 @@ -0,0 +1,758 @@
    14.4 +#
    14.5 +# <meta:header>
    14.6 +#   <meta:licence>
    14.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
    14.8 +#
    14.9 +#     This information is free software: you can redistribute it and/or modify
   14.10 +#     it under the terms of the GNU General Public License as published by
   14.11 +#     the Free Software Foundation, either version 3 of the License, or
   14.12 +#     (at your option) any later version.
   14.13 +#
   14.14 +#     This information is distributed in the hope that it will be useful,
   14.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   14.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   14.17 +#     GNU General Public License for more details.
   14.18 +#
   14.19 +#     You should have received a copy of the GNU General Public License
   14.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   14.21 +#   </meta:licence>
   14.22 +# </meta:header>
   14.23 +#
   14.24 +#
   14.25 +
   14.26 +# -----------------------------------------------------
   14.27 +# Create a new VM.
   14.28 +#[user@trop01]
   14.29 +
   14.30 +    createvm
   14.31 +
   14.32 +    >   INFO : Node name [Astoalith]
   14.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
   14.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
   14.35 +    >   INFO : Disc name [Astoalith.qcow]
   14.36 +    >   INFO : Disc size [16GiB]
   14.37 +
   14.38 +    vmname=Astoalith
   14.39 +
   14.40 +
   14.41 +# -----------------------------------------------------
   14.42 +# Define a hosts lookup function.
   14.43 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
   14.44 +#[user@trop01]
   14.45 +
   14.46 +    getipv4()
   14.47 +        {
   14.48 +        getent hosts "${1:?}" | cut -d ' ' -f 1
   14.49 +        }
   14.50 +
   14.51 +
   14.52 +#---------------------------------------------------------------------
   14.53 +# Update the ssh keys for the VM.
   14.54 +#[user@trop01]
   14.55 +
   14.56 +    # Remove the current key.
   14.57 +    ssh-keygen -q -R "${vmname:?}"
   14.58 +
   14.59 +    # Add the host key(s) to known_hosts
   14.60 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
   14.61 +
   14.62 +    # Add the IP address key(s) to known_hosts
   14.63 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
   14.64 +
   14.65 +
   14.66 +# -----------------------------------------------------
   14.67 +# Login as Stevedore
   14.68 +#[user@trop]
   14.69 +
   14.70 +    ssh "${vmname:?}"
   14.71 +
   14.72 +
   14.73 +# -----------------------------------------------------
   14.74 +# Install our secret function.
   14.75 +#[user@virtual]
   14.76 +
   14.77 +    mkdir "${HOME:?}/bin"
   14.78 +    cat > "${HOME:?}/bin/secret" << 'EOF'
   14.79 +#!/bin/sh
   14.80 +ssh -n \
   14.81 +    'dave@shepseskaf.roe.ac.uk' \
   14.82 +    "bin/secret '${1}'"
   14.83 +EOF
   14.84 +
   14.85 +    chmod u+x "${HOME:?}/bin/secret"
   14.86 +    secret 'frog'
   14.87 +
   14.88 +
   14.89 +# -----------------------------------------------------
   14.90 +# Download our builder compose file
   14.91 +#[user@virtual]
   14.92 +
   14.93 +    wget -O builder.yml \
   14.94 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
   14.95 +
   14.96 +
   14.97 +# -----------------------------------------------------
   14.98 +# Set the target branch
   14.99 +#[user@virtual]
  14.100 +
  14.101 +    branch=2.1.23-zrq-update-depends
  14.102 +    branch=2.1.24-zrq-userdata-drop
  14.103 +    branch=2.1.25-zrq-tap-controller
  14.104 +    branch=2.1.31-stv-cleanup
  14.105 +    branch=2.1.32-zrq-thread-pools
  14.106 +
  14.107 +
  14.108 +# -----------------------------------------------------
  14.109 +# Run our builder.
  14.110 +#[user@virtual]
  14.111 +
  14.112 +    export branch
  14.113 +    export secretsh=$(which secret)
  14.114 +
  14.115 +    docker-compose \
  14.116 +        --file "builder.yml" \
  14.117 +        run \
  14.118 +            builder
  14.119 +
  14.120 +    # -----------------------------------------------------
  14.121 +    # Test our secret function.
  14.122 +    #[root@builder]
  14.123 +
  14.124 +        secret 'frog'
  14.125 +
  14.126 +    # -----------------------------------------------------
  14.127 +    # Initialise our paths.
  14.128 +    #[root@builder]
  14.129 +
  14.130 +        PATH=${PATH}:/builder/bin
  14.131 +
  14.132 +        : ${FIRETHORN_HOME:=/etc/firethorn}
  14.133 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
  14.134 +
  14.135 +        export FIRETHORN_HOME
  14.136 +        export FIRETHORN_CODE
  14.137 +
  14.138 +    # -----------------------------------------------------
  14.139 +    # Checkout a copy of our source code.
  14.140 +    #[root@builder]
  14.141 +
  14.142 +        02.01-checkout.sh
  14.143 +
  14.144 +    # -----------------------------------------------------
  14.145 +    # Get the buildtag from our branch.
  14.146 +    #[root@builder]
  14.147 +
  14.148 +        pushd "${FIRETHORN_CODE:?}"
  14.149 +
  14.150 +            source 'bin/util.sh'
  14.151 +            export buildtag=$(getbuildtag)
  14.152 +
  14.153 +        popd
  14.154 +
  14.155 +
  14.156 +
  14.157 +#
  14.158 +# OGSA-DAI webapp fails to initialize.
  14.159 +# Lots of ArrayIndexOutOfBoundsException ..
  14.160 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
  14.161 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
  14.162 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
  14.163 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  14.164 +
  14.165 +    # -----------------------------------------------------
  14.166 +    # Revert the base image ?
  14.167 +    #[root@builder]
  14.168 +
  14.169 +        vi /var/local/build/firethorn/docker/fedora/Dockerfile
  14.170 +
  14.171 +        -   # FROM latest (fedora:29)
  14.172 +        -   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
  14.173 +        -   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
  14.174 +
  14.175 +        +   # FROM fedora:28
  14.176 +        +   # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
  14.177 +        +   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
  14.178 +
  14.179 +
  14.180 +    # -----------------------------------------------------
  14.181 +    # Revert the Tomcat version ?
  14.182 +    #[root@builder]
  14.183 +
  14.184 +        vi /var/local/build/firethorn/docker/tomcat/Dockerfile
  14.185 +
  14.186 +        -   ENV TOMCAT_VERSION 9.0.20
  14.187 +        +   ENV TOMCAT_VERSION 9.0.10
  14.188 +
  14.189 +
  14.190 +        vi /var/local/build/firethorn/pom.xml
  14.191 +
  14.192 +        -   <tomcat.version>9.0.20</tomcat.version>
  14.193 +        +   <tomcat.version>9.0.20</tomcat.version>
  14.194 +
  14.195 +
  14.196 +    # -----------------------------------------------------
  14.197 +    # Revert the Java dependencies ?
  14.198 +    #[root@builder]
  14.199 +
  14.200 +        vi /var/local/build/firethorn/pom.xml
  14.201 +
  14.202 +            <dependency>
  14.203 +                <groupId>commons-codec</groupId>
  14.204 +                <artifactId>commons-codec</artifactId>
  14.205 +        -       <version>1.12</version>
  14.206 +        +       <version>1.11</version>
  14.207 +            </dependency>
  14.208 +
  14.209 +            <dependency>
  14.210 +                <groupId>org.apache.commons</groupId>
  14.211 +                <artifactId>commons-lang3</artifactId>
  14.212 +        -       <version>3.9</version>
  14.213 +        +       <version>3.7</version>
  14.214 +            </dependency>
  14.215 +
  14.216 +            <dependency>
  14.217 +                <groupId>commons-fileupload</groupId>
  14.218 +                <artifactId>commons-fileupload</artifactId>
  14.219 +        -       <version>1.4</version>
  14.220 +        +       <version>1.3.3</version>
  14.221 +            </dependency>
  14.222 +
  14.223 +            <dependency>
  14.224 +                <groupId>org.apache.httpcomponents</groupId>
  14.225 +                <artifactId>httpclient</artifactId>
  14.226 +        -       <version>4.5.8</version>
  14.227 +        +       <version>4.5.6</version>
  14.228 +            </dependency>
  14.229 +
  14.230 +            <dependency>
  14.231 +                <groupId>com.google.guava</groupId>
  14.232 +                <artifactId>guava</artifactId>
  14.233 +        -       <version>27.1-jre</version>
  14.234 +        +       <version>26.0-jre</version>
  14.235 +            </dependency>
  14.236 +
  14.237 +            <dependency>
  14.238 +                <groupId>joda-time</groupId>
  14.239 +                <artifactId>joda-time</artifactId>
  14.240 +        -       <version>2.12</version>
  14.241 +        +       <version>2.10</version>
  14.242 +            </dependency>
  14.243 +
  14.244 +            <dependency>
  14.245 +                <groupId>org.jasypt</groupId>
  14.246 +                <artifactId>jasypt</artifactId>
  14.247 +        -       <version>1.9.3</version>
  14.248 +        +       <version>1.9.2</version>
  14.249 +            </dependency>
  14.250 +
  14.251 +            <dependency>
  14.252 +                <groupId>org.projectlombok</groupId>
  14.253 +                <artifactId>lombok</artifactId>
  14.254 +        -       <version>1.18.8</version>
  14.255 +        +       <version>1.18.2</version>
  14.256 +            </dependency>
  14.257 +
  14.258 +    # -----------------------------------------------------
  14.259 +    # Revert the Spring dependencies ?
  14.260 +    #[root@builder]
  14.261 +
  14.262 +        vi /var/local/build/firethorn/firethorn-spring/pom.xml
  14.263 +
  14.264 +            <properties>
  14.265 +
  14.266 +        -       <spring.version>5.1.7.RELEASE</spring.version>
  14.267 +        +       <spring.version>5.0.8.RELEASE</spring.version>
  14.268 +
  14.269 +        -       <hibernate.version>5.4.3.Final</hibernate.version>
  14.270 +        +       <hibernate.version>5.3.1.Final</hibernate.version>
  14.271 +
  14.272 +        -       <hibernate.validator>6.0.16.Final</hibernate.validator>
  14.273 +        +       <hibernate.validator>6.0.11.Final</hibernate.validator>
  14.274 +
  14.275 +        -       <aspectj.version>1.9.4</aspectj.version>
  14.276 +        +       <aspectj.version>1.9.1</aspectj.version>
  14.277 +
  14.278 +        -       <jackson.version>2.9.9</jackson.version>
  14.279 +        +       <jackson.version>2.9.6</jackson.version>
  14.280 +
  14.281 +            </properties>
  14.282 +
  14.283 +            ....
  14.284 +            ....
  14.285 +
  14.286 +            <dependency>
  14.287 +                <groupId>cglib</groupId>
  14.288 +                <artifactId>cglib</artifactId>
  14.289 +        -       <version>3.2.12</version>
  14.290 +        +       <version>3.2.7</version>
  14.291 +            </dependency>
  14.292 +
  14.293 +
  14.294 +    # -----------------------------------------------------
  14.295 +    # Revert the logging dependencies ?
  14.296 +    #[root@builder]
  14.297 +
  14.298 +        vi /var/local/build/firethorn/firethorn-logging/pom.xml
  14.299 +
  14.300 +            <properties>
  14.301 +                ....
  14.302 +
  14.303 +        -       <org.slf4j.version>1.7.26</org.slf4j.version>
  14.304 +        +       <org.slf4j.version>1.7.25</org.slf4j.version>
  14.305 +
  14.306 +                ....
  14.307 +            </properties>
  14.308 +
  14.309 +
  14.310 +    # -----------------------------------------------------
  14.311 +    # Revert the JDBC dependencies ?
  14.312 +    #[root@builder]
  14.313 +
  14.314 +        vi /var/local/build/firethorn/firethorn-jdbc/pom.xml
  14.315 +
  14.316 +            <properties>
  14.317 +                ....
  14.318 +
  14.319 +                <dependency>
  14.320 +                    <groupId>org.hsqldb</groupId>
  14.321 +                    <artifactId>hsqldb</artifactId>
  14.322 +        -           <version>2.5.0</version>
  14.323 +        +           <version>2.4.1</version>
  14.324 +                </dependency>
  14.325 +
  14.326 +                <dependency>
  14.327 +                    <groupId>org.postgresql</groupId>
  14.328 +                    <artifactId>postgresql</artifactId>
  14.329 +        -           <version>42.2.5</version>
  14.330 +        +           <version>42.2.4</version>
  14.331 +                </dependency>
  14.332 +
  14.333 +                <dependency>
  14.334 +                    <groupId>mysql</groupId>
  14.335 +                    <artifactId>mysql-connector-java</artifactId>
  14.336 +        -           <version>8.0.16</version>
  14.337 +        +           <version>8.0.12</version>
  14.338 +                </dependency>
  14.339 +
  14.340 +                <dependency>
  14.341 +                    <groupId>org.mariadb.jdbc</groupId>
  14.342 +                    <artifactId>mariadb-java-client</artifactId>
  14.343 +        +           <version>2.4.1</version>
  14.344 +        -           <version>2.2.6</version>
  14.345 +                </dependency>
  14.346 +
  14.347 +                ....
  14.348 +            </properties>
  14.349 +
  14.350 +    # -----------------------------------------------------
  14.351 +    # Edit the webapp Thread pools.
  14.352 +    #[root@builder]
  14.353 +
  14.354 +        vi /var/local/build/firethorn/firethorn-webapp/src/main/docker/firethorn/conf/server.xml
  14.355 +
  14.356 +            <Executor
  14.357 +                name="main-interface"
  14.358 +                namePrefix="main-interface-"
  14.359 +        -       maxThreads="20"
  14.360 +        +       maxThreads="10"
  14.361 +                minSpareThreads="5"
  14.362 +                />
  14.363 +
  14.364 +            <Executor
  14.365 +                name="callback-interface"
  14.366 +                namePrefix="callback-interface-"
  14.367 +                maxThreads="20"
  14.368 +                minSpareThreads="5"
  14.369 +                />
  14.370 +
  14.371 +
  14.372 +    # -----------------------------------------------------
  14.373 +    # Build our base images.
  14.374 +    #[root@builder]
  14.375 +
  14.376 +        04.01-buildbase.sh
  14.377 +
  14.378 +    # -----------------------------------------------------
  14.379 +    # Compile our Java code.
  14.380 +    #[root@builder]
  14.381 +
  14.382 +        05.01-javamaven.sh
  14.383 +
  14.384 +    # -----------------------------------------------------
  14.385 +    # Build our Java containers.
  14.386 +    #[root@builder]
  14.387 +
  14.388 +        05.02-javadocker.sh
  14.389 +
  14.390 +    # -----------------------------------------------------
  14.391 +    # Fetch our Python code.
  14.392 +    #[root@builder]
  14.393 +
  14.394 +        echo "Fetching client source"
  14.395 +
  14.396 +        ftpysrc='/var/local/build/client'
  14.397 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
  14.398 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
  14.399 +
  14.400 +        if [  -e "${ftpysrc:?}" ]
  14.401 +        then
  14.402 +            pushd "${ftpysrc:?}"
  14.403 +
  14.404 +                echo "Updating Python code"
  14.405 +                git pull
  14.406 +
  14.407 +            popd
  14.408 +        else
  14.409 +            pushd "$(dirname ${ftpysrc:?})"
  14.410 +
  14.411 +                echo "Cloning Python code from [${ftpygit:?}]"
  14.412 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
  14.413 +
  14.414 +            popd
  14.415 +        fi
  14.416 +
  14.417 +    # -----------------------------------------------------
  14.418 +    # Delete the embedded metadata.
  14.419 +    #[root@builder]
  14.420 +
  14.421 +        pushd "${ftpysrc:?}"
  14.422 +
  14.423 +            du -h  firethorn/meta
  14.424 +            rm -rf firethorn/meta
  14.425 +
  14.426 +        popd
  14.427 +
  14.428 +    # -----------------------------------------------------
  14.429 +    # Build our Python container.
  14.430 +    #[root@builder]
  14.431 +
  14.432 +        echo "Building client image"
  14.433 +
  14.434 +        pushd "${FIRETHORN_CODE:?}"
  14.435 +
  14.436 +            export buildsrc=$(pwd)
  14.437 +            export ftpysrc
  14.438 +
  14.439 +            docker-compose \
  14.440 +                --file docker/compose/client/firethorn-py.yml \
  14.441 +                build
  14.442 +
  14.443 +        popd
  14.444 +
  14.445 +
  14.446 +    # -----------------------------------------------------
  14.447 +    # Create our chain properties.
  14.448 +    #[user@builder]
  14.449 +
  14.450 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
  14.451 +
  14.452 +            buildtag=${buildtag:?}
  14.453 +
  14.454 +            metadata=$(pwgen 20 1)
  14.455 +            metauser=$(pwgen 20 1)
  14.456 +            metapass=$(pwgen 20 1)
  14.457 +
  14.458 +            usertype=mssql
  14.459 +            userhost=$(secret 'firethorn.user.host')
  14.460 +            userdata=$(secret 'firethorn.user.data')
  14.461 +            useruser=$(secret 'firethorn.user.user')
  14.462 +            userpass=$(secret 'firethorn.user.pass')
  14.463 +
  14.464 +            datatype=mssql
  14.465 +            datahost=$(secret 'firethorn.data.host')
  14.466 +            datadata=$(secret 'firethorn.data.data')
  14.467 +            datauser=$(secret 'firethorn.data.user')
  14.468 +            datapass=$(secret 'firethorn.data.pass')
  14.469 +
  14.470 +            tunneluser=$(secret 'ssh.tunnel.user')
  14.471 +            tunnelhost=$(secret 'ssh.tunnel.host')
  14.472 +
  14.473 +            admingroup=Hyaenidae
  14.474 +            adminuser=Aardwolf
  14.475 +            adminpass=$(pwgen 20 1)
  14.476 +
  14.477 +            guestgroup=Afrotheria
  14.478 +            guestuser=Hyrax
  14.479 +            guestpass=$(pwgen 20 1)
  14.480 +
  14.481 +            tapresource=Wilhelmina
  14.482 +            tapschemadata=data-$(pwgen 10 1)
  14.483 +            tapschemauser=user-$(pwgen 10 1)
  14.484 +            tapschemapass=pass-$(pwgen 10 1)
  14.485 +
  14.486 +EOF
  14.487 +
  14.488 +    # -----------------------------------------------------
  14.489 +    # Determine our location.
  14.490 +    #[user@builder]
  14.491 +
  14.492 +    # 'local' if VM is inside UoE
  14.493 +    # 'remote' if VM is at outside UoE
  14.494 +    # TODO Change these to 'internal' and 'external'
  14.495 +
  14.496 +        external=$(curl -4 --silent 'http://icanhazip.com/')
  14.497 +
  14.498 +        EDINBURGH='129.215.*'
  14.499 +
  14.500 +        if [[ ${external:?} == ${EDINBURGH} ]]
  14.501 +        then
  14.502 +            echo "Address [${external:?}] is in Edinburgh"
  14.503 +            #location=internal
  14.504 +            location=local
  14.505 +        else
  14.506 +            echo "Address [${external:?}] is not in Edinburgh"
  14.507 +            #location=external
  14.508 +            location=remote
  14.509 +        fi
  14.510 +
  14.511 +
  14.512 +    # -----------------------------------------------------
  14.513 +    # Link our compose config and start our test containers ...
  14.514 +    #[user@builder]
  14.515 +
  14.516 +        # Check the env file is linked !!
  14.517 +
  14.518 +        pushd "${FIRETHORN_HOME:?}"
  14.519 +
  14.520 +            ln -sf 'chain.properties' '.env'
  14.521 +
  14.522 +            docker-compose \
  14.523 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
  14.524 +                run \
  14.525 +                    angela
  14.526 +
  14.527 +
  14.528 +# -----------------------------------------------------
  14.529 +# -----------------------------------------------------
  14.530 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
  14.531 +#[user@virtual]
  14.532 +
  14.533 +    sudo -s
  14.534 +
  14.535 +    container=baryptera_gillian_1
  14.536 +
  14.537 +    pushd $(
  14.538 +        docker inspect \
  14.539 +            "${container:?}" \
  14.540 +      | jq -r '
  14.541 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  14.542 +            '
  14.543 +            )
  14.544 +
  14.545 +    tail -f firethorn-debug.log
  14.546 +
  14.547 +# -----------------------------------------------------
  14.548 +# -----------------------------------------------------
  14.549 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
  14.550 +#[user@virtual]
  14.551 +
  14.552 +    sudo -s
  14.553 +
  14.554 +    container=baryptera_jarmila_1
  14.555 +
  14.556 +    pushd $(
  14.557 +        docker inspect \
  14.558 +            "${container:?}" \
  14.559 +      | jq -r '
  14.560 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  14.561 +            '
  14.562 +            )
  14.563 +
  14.564 +    tail -f ogsadai.log
  14.565 +
  14.566 +# -----------------------------------------------------
  14.567 +# -----------------------------------------------------
  14.568 +# Run our Python tests ...
  14.569 +#[user@python]
  14.570 +
  14.571 +import os
  14.572 +import uuid
  14.573 +import time
  14.574 +import firethorn as ftpy
  14.575 +
  14.576 +#
  14.577 +# Create our firethorn client (using named param).
  14.578 +firethorn = ftpy.Firethorn(
  14.579 +    endpoint = os.environ.get(
  14.580 +        'endpoint'
  14.581 +        )
  14.582 +    )
  14.583 +
  14.584 +#
  14.585 +# Login as the admin account.
  14.586 +firethorn.login(
  14.587 +    os.environ.get('adminuser'),
  14.588 +    os.environ.get('adminpass'),
  14.589 +    os.environ.get('admingroup')
  14.590 +    )
  14.591 +
  14.592 +#
  14.593 +# Create a JdbcResource to connect to the ATLAS database.
  14.594 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
  14.595 +    "ATLAS JDBC resource",
  14.596 +    os.environ.get('datadata'),
  14.597 +    '*',
  14.598 +    os.environ.get('datatype'),
  14.599 +    os.environ.get('datahost'),
  14.600 +    os.environ.get('datauser'),
  14.601 +    os.environ.get('datapass')
  14.602 +    )
  14.603 +print(
  14.604 +    atlas_jdbc
  14.605 +    )
  14.606 +
  14.607 +#
  14.608 +# Create an AdqlResource to represent the JdbcResource.
  14.609 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
  14.610 +    "ATLAS ADQL resource"
  14.611 +    )
  14.612 +print(
  14.613 +    atlas_adql
  14.614 +    )
  14.615 +
  14.616 +#
  14.617 +# Import the target JdbcSchema into AdqlSchema.
  14.618 +schema_names = [
  14.619 +    "ATLASDR1"
  14.620 +    ]
  14.621 +
  14.622 +for schema_name in schema_names:
  14.623 +    print(schema_name)
  14.624 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
  14.625 +        schema_name,
  14.626 +        "dbo"
  14.627 +        )
  14.628 +    if (None != jdbc_schema):
  14.629 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
  14.630 +        adql_schema = atlas_adql.import_jdbc_schema(
  14.631 +            jdbc_schema,
  14.632 +            schema_name,
  14.633 +            metadoc=metadoc
  14.634 +            )
  14.635 +
  14.636 +#
  14.637 +# Admin user
  14.638 +# -------- -------- -------- --------
  14.639 +# Normal user
  14.640 +#
  14.641 +
  14.642 +#
  14.643 +# Login using a guest account.
  14.644 +firethorn.login(
  14.645 +    str(uuid.uuid4()),
  14.646 +    str(uuid.uuid4()),
  14.647 +    None
  14.648 +    )
  14.649 +
  14.650 +#
  14.651 +# Create a new workspace.
  14.652 +workspace = firethorn.firethorn_engine.create_adql_resource(
  14.653 +    "Query resource"
  14.654 +    )
  14.655 +
  14.656 +#
  14.657 +# Import the ATLAS schemas into our workspace
  14.658 +for schema in atlas_adql.select_schemas():
  14.659 +    workspace.import_adql_schema(
  14.660 +        schema
  14.661 +        )
  14.662 +
  14.663 +#
  14.664 +# Create and run a query.
  14.665 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  14.666 +query_obj = workspace.create_query(
  14.667 +    query_str,
  14.668 +    "COMPLETED",
  14.669 +    None,
  14.670 +    3000000
  14.671 +    )
  14.672 +print(
  14.673 +    query_obj
  14.674 +    )
  14.675 +print(
  14.676 +    query_obj.table()
  14.677 +    )
  14.678 +print(
  14.679 +    query_obj.table().count()
  14.680 +    )
  14.681 +
  14.682 +#
  14.683 +# Iterate the metadata tree
  14.684 +for schema in atlas_adql.select_schemas():
  14.685 +    for table in schema.select_tables():
  14.686 +        print(
  14.687 +            "table  [{}][{}]".format(
  14.688 +                schema.name(),
  14.689 +                table.name()
  14.690 +                )
  14.691 +            )
  14.692 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
  14.693 +            schema.name(),
  14.694 +            table.name()
  14.695 +            )
  14.696 +        query_obj = workspace.create_query(
  14.697 +            query_str,
  14.698 +            "COMPLETED",
  14.699 +            None,
  14.700 +            3000000
  14.701 +            )
  14.702 +        py_table = query_obj.table().as_astropy()
  14.703 +        py_table.pprint()
  14.704 +
  14.705 +#
  14.706 +# Run some queries in parallel
  14.707 +from concurrent.futures import ThreadPoolExecutor
  14.708 +import concurrent.futures
  14.709 +from datetime import datetime
  14.710 +
  14.711 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
  14.712 +
  14.713 +def do_query(workspace, query_str, limit, delay):
  14.714 +    before = datetime.now()
  14.715 +    query_obj = workspace.create_query(
  14.716 +        query_str,
  14.717 +        "COMPLETED",
  14.718 +        None,
  14.719 +        200000,
  14.720 +            {
  14.721 +            "adql.query.limit.rows"  : limit,
  14.722 +            "adql.query.delay.every" : delay
  14.723 +            }
  14.724 +        )
  14.725 +    after = datetime.now()
  14.726 +    return (
  14.727 +        (after - before),
  14.728 +        query_obj.json_object.get("results").get("count")
  14.729 +        )
  14.730 +
  14.731 +def do_queries(workspace, query_str, threads, delay):
  14.732 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
  14.733 +        futures = {
  14.734 +            executor.submit(
  14.735 +                do_query,
  14.736 +                workspace,
  14.737 +                query_str,
  14.738 +                limit,
  14.739 +                delay
  14.740 +                ): limit for limit in range(threads, 0, -1)
  14.741 +            }
  14.742 +        for future in concurrent.futures.as_completed(futures):
  14.743 +            print(
  14.744 +                future.result()[0],
  14.745 +                ':',
  14.746 +                future.result()[1]
  14.747 +                )
  14.748 +
  14.749 +for loop in range(1, 4):
  14.750 +    for threads in range(1, 50):
  14.751 +        for delay in range(1000, -100, -100):
  14.752 +            print("---- ", loop, threads, delay)
  14.753 +            do_queries(
  14.754 +                workspace,
  14.755 +                query_str,
  14.756 +                threads,
  14.757 +                delay
  14.758 +                )
  14.759 +
  14.760 +
  14.761 +
    15.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    15.2 +++ b/doc/notes/zrq/20190605-01-test-build.txt	Tue Jul 02 04:12:49 2019 +0200
    15.3 @@ -0,0 +1,547 @@
    15.4 +#
    15.5 +# <meta:header>
    15.6 +#   <meta:licence>
    15.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
    15.8 +#
    15.9 +#     This information is free software: you can redistribute it and/or modify
   15.10 +#     it under the terms of the GNU General Public License as published by
   15.11 +#     the Free Software Foundation, either version 3 of the License, or
   15.12 +#     (at your option) any later version.
   15.13 +#
   15.14 +#     This information is distributed in the hope that it will be useful,
   15.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   15.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15.17 +#     GNU General Public License for more details.
   15.18 +#
   15.19 +#     You should have received a copy of the GNU General Public License
   15.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   15.21 +#   </meta:licence>
   15.22 +# </meta:header>
   15.23 +#
   15.24 +#
   15.25 +
   15.26 +# -----------------------------------------------------
   15.27 +# Login as to the VM.
   15.28 +#[user@trop]
   15.29 +
   15.30 +    ssh "${vmname:?}"
   15.31 +
   15.32 +
   15.33 +# -----------------------------------------------------
   15.34 +# Set the target branch
   15.35 +#[user@virtual]
   15.36 +
   15.37 +    branch=2.1.32-zrq-thread-pools
   15.38 +
   15.39 +
   15.40 +# -----------------------------------------------------
   15.41 +# Run our builder.
   15.42 +#[user@virtual]
   15.43 +
   15.44 +    export branch
   15.45 +    export secretsh=$(which secret)
   15.46 +
   15.47 +    docker-compose \
   15.48 +        --file "builder.yml" \
   15.49 +        run \
   15.50 +            builder
   15.51 +
   15.52 +    # -----------------------------------------------------
   15.53 +    # Test our secret function.
   15.54 +    #[root@builder]
   15.55 +
   15.56 +        secret 'frog'
   15.57 +
   15.58 +        >   Green Frog
   15.59 +
   15.60 +    # -----------------------------------------------------
   15.61 +    # Initialise our paths.
   15.62 +    #[root@builder]
   15.63 +
   15.64 +        PATH=${PATH}:/builder/bin
   15.65 +
   15.66 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   15.67 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   15.68 +
   15.69 +        export FIRETHORN_HOME
   15.70 +        export FIRETHORN_CODE
   15.71 +
   15.72 +
   15.73 +    # -----------------------------------------------------
   15.74 +    # Update our source code.
   15.75 +    #[root@builder]
   15.76 +
   15.77 +        02.01-checkout.sh
   15.78 +
   15.79 +        >   /var/local/build/firethorn /
   15.80 +        >   Updating source code from []
   15.81 +        >   pulling from http://wfau.metagrid.co.uk/code/firethorn
   15.82 +        >   searching for changes
   15.83 +        >   no changes found
   15.84 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   15.85 +        >   /
   15.86 +        >   /var/local/build/firethorn /
   15.87 +        >   Updating branch [2.1.32-zrq-thread-pools]
   15.88 +        >   0 files updated, 0 files merged, 0 files removed, 0 files unresolved
   15.89 +        >   /
   15.90 +
   15.91 +
   15.92 +# OGSA-DAI webapp fails to initialize.
   15.93 +# Lots of ArrayIndexOutOfBoundsException ..
   15.94 +# 2019-06-04 01:48:36,506 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] **** inside initializeWebapp
   15.95 +# 2019-06-04 01:48:36,507 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@562ec1f6: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
   15.96 +# 2019-06-04 01:48:36,511 DEBUG context.OGSADAIContext [http-nio-8080-exec-5,debug:84] Initializing context...
   15.97 +# 2019-06-04 01:48:36,531 FATAL rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-5,fatalExceptionAndChildren:458] #1559612916531:10# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
   15.98 +
   15.99 +    # -----------------------------------------------------
  15.100 +    # Update the base image back to Fedora 29.
  15.101 +    # PASS that works
  15.102 +    #[root@builder]
  15.103 +
  15.104 +        pushd ${FIRETHORN_CODE:?}
  15.105 +
  15.106 +            vi docker/fedora/Dockerfile
  15.107 +
  15.108 +                # FROM latest (fedora:29)
  15.109 +                # https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
  15.110 +            +   FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
  15.111 +            -   FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
  15.112 +
  15.113 +
  15.114 +    # -----------------------------------------------------
  15.115 +    # Change the Tomcat version back to 9.0.20.
  15.116 +    #[root@builder]
  15.117 +
  15.118 +            vi docker/tomcat/Dockerfile
  15.119 +
  15.120 +            -   ENV TOMCAT_VERSION 9.0.10
  15.121 +            +   ENV TOMCAT_VERSION 9.0.20
  15.122 +
  15.123 +            vi pom.xml
  15.124 +
  15.125 +            -   <tomcat.version>9.0.10</tomcat.version>
  15.126 +            +   <tomcat.version>9.0.20</tomcat.version>
  15.127 +
  15.128 +#
  15.129 +# THAT causes the ArrayIndexOutOfBoundsException in OGSA-DAI.
  15.130 +2019-06-06 03:28:50,237 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
  15.131 +2019-06-06 03:28:50,572 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559791730572:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.132 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.133 +2019-06-06 03:28:50,573 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.134 +2019-06-06 03:28:50,574 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559791730573:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.135 +2019-06-06 03:29:12,403 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] **** inside initializeWebapp
  15.136 +2019-06-06 03:29:12,407 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-2,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@7a59a6f2: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
  15.137 +
  15.138 +    # -----------------------------------------------------
  15.139 +    # Change the Tomcat version back to 9.0.10.
  15.140 +    #[root@builder]
  15.141 +
  15.142 +            vi docker/tomcat/Dockerfile
  15.143 +
  15.144 +            +   ENV TOMCAT_VERSION 9.0.10
  15.145 +            -   ENV TOMCAT_VERSION 9.0.20
  15.146 +
  15.147 +            vi pom.xml
  15.148 +
  15.149 +            +   <tomcat.version>9.0.10</tomcat.version>
  15.150 +            -   <tomcat.version>9.0.20</tomcat.version>
  15.151 +
  15.152 +    # -----------------------------------------------------
  15.153 +    # Revert the Java dependencies.
  15.154 +    #[root@builder]
  15.155 +
  15.156 +        hg status
  15.157 +
  15.158 +        >   M docker/tomcat/Dockerfile
  15.159 +        >   M pom.xml
  15.160 +        >   M firethorn-jdbc/pom.xml
  15.161 +        >   M firethorn-logging/pom.xml
  15.162 +        >   M firethorn-spring/pom.xml
  15.163 +        >   M firethorn-webapp/src/main/docker/firethorn/conf/server.xml
  15.164 +
  15.165 +        hg revert pom.xml
  15.166 +        hg revert firethorn-jdbc/pom.xml
  15.167 +        hg revert firethorn-logging/pom.xml
  15.168 +        hg revert firethorn-spring/pom.xml
  15.169 +
  15.170 +    # -----------------------------------------------------
  15.171 +    # List the remaining changes.
  15.172 +    #[root@builder]
  15.173 +
  15.174 +        hg diff pom.xml
  15.175 +
  15.176 +            -        <tomcat.version>9.0.20</tomcat.version>
  15.177 +            +        <tomcat.version>9.0.10</tomcat.version>
  15.178 +
  15.179 +
  15.180 +        hg diff docker/tomcat/Dockerfile
  15.181 +
  15.182 +            -   ENV TOMCAT_VERSION 9.0.20
  15.183 +            +   ENV TOMCAT_VERSION 9.0.10
  15.184 +
  15.185 +
  15.186 +        hg diff firethorn-webapp/src/main/docker/firethorn/conf/server.xml
  15.187 +
  15.188 +                 <Executor
  15.189 +                     name="main-interface"
  15.190 +                     namePrefix="main-interface-"
  15.191 +            -        maxThreads="20"
  15.192 +            +        maxThreads="10"
  15.193 +                     minSpareThreads="5"
  15.194 +                     />
  15.195 +
  15.196 +#
  15.197 +# Still get the ArrayIndexOutOfBoundsException in OGSA-DAI.
  15.198 +2019-06-06 04:09:58,633 DEBUG context.OGSADAIContext [main,debug:84] Initializing context...
  15.199 +2019-06-06 04:09:59,070 FATAL rest.JerseyStaticOGSADAIContextInitializer [main,fatalExceptionAndChildren:458] #1559794199070:0# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.200 +2019-06-06 04:09:59,071 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.IllegalArgumentException: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.201 +2019-06-06 04:09:59,072 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.202 +2019-06-06 04:09:59,073 ERROR rest.JerseyRootService [main,errorExceptionAndChildren:401] #1559794199071:1# java.lang.ArrayIndexOutOfBoundsException: Index 66158 out of bounds for length 30082
  15.203 +2019-06-06 04:10:20,549 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] **** inside initializeWebapp
  15.204 +2019-06-06 04:10:20,553 DEBUG rest.JerseyStaticOGSADAIContextInitializer [http-nio-8080-exec-1,debug:84] genericCtx = org.springframework.context.support.GenericApplicationContext@3a7fcd66: startup date [Thu Jan 01 00:00:00 UTC 1970]; root of context hierarchy
  15.205 +
  15.206 +
  15.207 +    # -----------------------------------------------------
  15.208 +    # Build our base images.
  15.209 +    #[root@builder]
  15.210 +
  15.211 +        04.01-buildbase.sh
  15.212 +
  15.213 +    # -----------------------------------------------------
  15.214 +    # Compile our Java code.
  15.215 +    #[root@builder]
  15.216 +
  15.217 +        05.01-javamaven.sh
  15.218 +
  15.219 +    # -----------------------------------------------------
  15.220 +    # Build our Java containers.
  15.221 +    #[root@builder]
  15.222 +
  15.223 +        05.02-javadocker.sh
  15.224 +
  15.225 +
  15.226 +    # -----------------------------------------------------
  15.227 +    # Build our Python client container.
  15.228 +    #[root@builder]
  15.229 +
  15.230 +        #
  15.231 +        # Python client container from yesterday is still in local build cache.
  15.232 +        #
  15.233 +
  15.234 +
  15.235 +    # -----------------------------------------------------
  15.236 +    # Create our chain properties.
  15.237 +    #[user@builder]
  15.238 +
  15.239 +        # Only need to do this once.
  15.240 +
  15.241 +        cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
  15.242 +
  15.243 +            buildtag=${buildtag:?}
  15.244 +
  15.245 +            metadata=$(pwgen 20 1)
  15.246 +            metauser=$(pwgen 20 1)
  15.247 +            metapass=$(pwgen 20 1)
  15.248 +
  15.249 +            usertype=mssql
  15.250 +            userhost=$(secret 'firethorn.user.host')
  15.251 +            userdata=$(secret 'firethorn.user.data')
  15.252 +            useruser=$(secret 'firethorn.user.user')
  15.253 +            userpass=$(secret 'firethorn.user.pass')
  15.254 +
  15.255 +            datatype=mssql
  15.256 +            datahost=$(secret 'firethorn.data.host')
  15.257 +            datadata=$(secret 'firethorn.data.data')
  15.258 +            datauser=$(secret 'firethorn.data.user')
  15.259 +            datapass=$(secret 'firethorn.data.pass')
  15.260 +
  15.261 +            tunneluser=$(secret 'ssh.tunnel.user')
  15.262 +            tunnelhost=$(secret 'ssh.tunnel.host')
  15.263 +
  15.264 +            admingroup=Hyaenidae
  15.265 +            adminuser=Aardwolf
  15.266 +            adminpass=$(pwgen 20 1)
  15.267 +
  15.268 +            guestgroup=Afrotheria
  15.269 +            guestuser=Hyrax
  15.270 +            guestpass=$(pwgen 20 1)
  15.271 +
  15.272 +            tapresource=Wilhelmina
  15.273 +            tapschemadata=data-$(pwgen 10 1)
  15.274 +            tapschemauser=user-$(pwgen 10 1)
  15.275 +            tapschemapass=pass-$(pwgen 10 1)
  15.276 +
  15.277 +EOF
  15.278 +
  15.279 +    # -----------------------------------------------------
  15.280 +    # Determine our location.
  15.281 +    #[user@builder]
  15.282 +
  15.283 +        # Only need to do this once.
  15.284 +
  15.285 +        external=$(curl -4 --silent 'http://icanhazip.com/')
  15.286 +
  15.287 +        EDINBURGH='129.215.*'
  15.288 +
  15.289 +        if [[ ${external:?} == ${EDINBURGH} ]]
  15.290 +        then
  15.291 +            echo "Address [${external:?}] is in Edinburgh"
  15.292 +            #location=internal
  15.293 +            location=local
  15.294 +        else
  15.295 +            echo "Address [${external:?}] is not in Edinburgh"
  15.296 +            #location=external
  15.297 +            location=remote
  15.298 +        fi
  15.299 +
  15.300 +
  15.301 +    # -----------------------------------------------------
  15.302 +    # Link our compose config and start our test containers ...
  15.303 +    #[user@builder]
  15.304 +
  15.305 +        # Check the env file is linked !!
  15.306 +
  15.307 +        pushd "${FIRETHORN_HOME:?}"
  15.308 +
  15.309 +            ln -sf 'chain.properties' '.env'
  15.310 +
  15.311 +            docker-compose \
  15.312 +                --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
  15.313 +                run \
  15.314 +                    angela
  15.315 +
  15.316 +
  15.317 +# -----------------------------------------------------
  15.318 +# -----------------------------------------------------
  15.319 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
  15.320 +#[user@virtual]
  15.321 +
  15.322 +    sudo -s
  15.323 +
  15.324 +    container=baryptera_gillian_1
  15.325 +
  15.326 +    pushd $(
  15.327 +        docker inspect \
  15.328 +            "${container:?}" \
  15.329 +      | jq -r '
  15.330 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  15.331 +            '
  15.332 +            )
  15.333 +
  15.334 +    tail -f firethorn-debug.log
  15.335 +
  15.336 +# -----------------------------------------------------
  15.337 +# -----------------------------------------------------
  15.338 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
  15.339 +#[user@virtual]
  15.340 +
  15.341 +    sudo -s
  15.342 +
  15.343 +    container=baryptera_jarmila_1
  15.344 +
  15.345 +    pushd $(
  15.346 +        docker inspect \
  15.347 +            "${container:?}" \
  15.348 +      | jq -r '
  15.349 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
  15.350 +            '
  15.351 +            )
  15.352 +
  15.353 +    tail -f ogsadai.log
  15.354 +
  15.355 +# -----------------------------------------------------
  15.356 +# -----------------------------------------------------
  15.357 +# Run our Python tests ...
  15.358 +#[user@python]
  15.359 +
  15.360 +import os
  15.361 +import uuid
  15.362 +import time
  15.363 +import firethorn as ftpy
  15.364 +
  15.365 +#
  15.366 +# Create our firethorn client (using named param).
  15.367 +firethorn = ftpy.Firethorn(
  15.368 +    endpoint = os.environ.get(
  15.369 +        'endpoint'
  15.370 +        )
  15.371 +    )
  15.372 +
  15.373 +#
  15.374 +# Login as the admin account.
  15.375 +firethorn.login(
  15.376 +    os.environ.get('adminuser'),
  15.377 +    os.environ.get('adminpass'),
  15.378 +    os.environ.get('admingroup')
  15.379 +    )
  15.380 +
  15.381 +#
  15.382 +# Create a JdbcResource to connect to the ATLAS database.
  15.383 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
  15.384 +    "ATLAS JDBC resource",
  15.385 +    os.environ.get('datadata'),
  15.386 +    '*',
  15.387 +    os.environ.get('datatype'),
  15.388 +    os.environ.get('datahost'),
  15.389 +    os.environ.get('datauser'),
  15.390 +    os.environ.get('datapass')
  15.391 +    )
  15.392 +print(
  15.393 +    atlas_jdbc
  15.394 +    )
  15.395 +
  15.396 +#
  15.397 +# Create an AdqlResource to represent the JdbcResource.
  15.398 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
  15.399 +    "ATLAS ADQL resource"
  15.400 +    )
  15.401 +print(
  15.402 +    atlas_adql
  15.403 +    )
  15.404 +
  15.405 +#
  15.406 +# Import the target JdbcSchema into AdqlSchema.
  15.407 +schema_names = [
  15.408 +    "ATLASDR1"
  15.409 +    ]
  15.410 +
  15.411 +for schema_name in schema_names:
  15.412 +    print(schema_name)
  15.413 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
  15.414 +        schema_name,
  15.415 +        "dbo"
  15.416 +        )
  15.417 +    if (None != jdbc_schema):
  15.418 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
  15.419 +        adql_schema = atlas_adql.import_jdbc_schema(
  15.420 +            jdbc_schema,
  15.421 +            schema_name,
  15.422 +            metadoc=metadoc
  15.423 +            )
  15.424 +
  15.425 +#
  15.426 +# Admin user
  15.427 +# -------- -------- -------- --------
  15.428 +# Normal user
  15.429 +#
  15.430 +
  15.431 +#
  15.432 +# Login using a guest account.
  15.433 +firethorn.login(
  15.434 +    str(uuid.uuid4()),
  15.435 +    str(uuid.uuid4()),
  15.436 +    None
  15.437 +    )
  15.438 +
  15.439 +#
  15.440 +# Create a new workspace.
  15.441 +workspace = firethorn.firethorn_engine.create_adql_resource(
  15.442 +    "Query resource"
  15.443 +    )
  15.444 +
  15.445 +#
  15.446 +# Import the ATLAS schemas into our workspace
  15.447 +for schema in atlas_adql.select_schemas():
  15.448 +    workspace.import_adql_schema(
  15.449 +        schema
  15.450 +        )
  15.451 +
  15.452 +#
  15.453 +# Create and run a query.
  15.454 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  15.455 +query_obj = workspace.create_query(
  15.456 +    query_str,
  15.457 +    "COMPLETED",
  15.458 +    None,
  15.459 +    3000000
  15.460 +    )
  15.461 +print(
  15.462 +    query_obj
  15.463 +    )
  15.464 +print(
  15.465 +    query_obj.table()
  15.466 +    )
  15.467 +print(
  15.468 +    query_obj.table().count()
  15.469 +    )
  15.470 +
  15.471 +#
  15.472 +# Iterate the metadata tree
  15.473 +for schema in atlas_adql.select_schemas():
  15.474 +    for table in schema.select_tables():
  15.475 +        print(
  15.476 +            "table  [{}][{}]".format(
  15.477 +                schema.name(),
  15.478 +                table.name()
  15.479 +                )
  15.480 +            )
  15.481 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
  15.482 +            schema.name(),
  15.483 +            table.name()
  15.484 +            )
  15.485 +        query_obj = workspace.create_query(
  15.486 +            query_str,
  15.487 +            "COMPLETED",
  15.488 +            None,
  15.489 +            3000000
  15.490 +            )
  15.491 +        py_table = query_obj.table().as_astropy()
  15.492 +        py_table.pprint()
  15.493 +
  15.494 +#
  15.495 +# Run some queries in parallel
  15.496 +from concurrent.futures import ThreadPoolExecutor
  15.497 +import concurrent.futures
  15.498 +from datetime import datetime
  15.499 +
  15.500 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
  15.501 +
  15.502 +def do_query(workspace, query_str, limit, delay):
  15.503 +    before = datetime.now()
  15.504 +    query_obj = workspace.create_query(
  15.505 +        query_str,
  15.506 +        "COMPLETED",
  15.507 +        None,
  15.508 +        200000,
  15.509 +            {
  15.510 +            "adql.query.limit.rows"  : limit,
  15.511 +            "adql.query.delay.every" : delay
  15.512 +            }
  15.513 +        )
  15.514 +    after = datetime.now()
  15.515 +    return (
  15.516 +        (after - before),
  15.517 +        query_obj.json_object.get("results").get("count")
  15.518 +        )
  15.519 +
  15.520 +def do_queries(workspace, query_str, threads, delay):
  15.521 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
  15.522 +        futures = {
  15.523 +            executor.submit(
  15.524 +                do_query,
  15.525 +                workspace,
  15.526 +                query_str,
  15.527 +                limit,
  15.528 +                delay
  15.529 +                ): limit for limit in range(threads, 0, -1)
  15.530 +            }
  15.531 +        for future in concurrent.futures.as_completed(futures):
  15.532 +            print(
  15.533 +                future.result()[0],
  15.534 +                ':',
  15.535 +                future.result()[1]
  15.536 +                )
  15.537 +
  15.538 +for loop in range(1, 10):
  15.539 +    for threads in range(1, 50):
  15.540 +        for delay in range(1000, -100, -100):
  15.541 +            print("---- ", loop, threads, delay)
  15.542 +            do_queries(
  15.543 +                workspace,
  15.544 +                query_str,
  15.545 +                threads,
  15.546 +                delay
  15.547 +                )
  15.548 +
  15.549 +
  15.550 +
    16.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    16.2 +++ b/doc/notes/zrq/20190606-01-database-errors.txt	Tue Jul 02 04:12:49 2019 +0200
    16.3 @@ -0,0 +1,26 @@
    16.4 +#
    16.5 +# <meta:header>
    16.6 +#   <meta:licence>
    16.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
    16.8 +#
    16.9 +#     This information is free software: you can redistribute it and/or modify
   16.10 +#     it under the terms of the GNU General Public License as published by
   16.11 +#     the Free Software Foundation, either version 3 of the License, or
   16.12 +#     (at your option) any later version.
   16.13 +#
   16.14 +#     This information is distributed in the hope that it will be useful,
   16.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
   16.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   16.17 +#     GNU General Public License for more details.
   16.18 +#
   16.19 +#     You should have received a copy of the GNU General Public License
   16.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
   16.21 +#   </meta:licence>
   16.22 +# </meta:header>
   16.23 +#
   16.24 +#
   16.25 +
   16.26 +
   16.27 +    2019-06-06 05:02:59,150 WARN  [main-interface-3] [JdbcResourceEntity] Exception while scanning catalog [53][UKIDSSDR4PLUS][Database 'UKIDSSDR4PLUS' cannot be opened due to inaccessible files or insufficient memory or disk space.  See the SQL Server errorlog for details.]
   16.28 +    2019-06-06 05:02:59,150 WARN  [main-interface-3] [SQLServerScanner] SQLException [945][S1000][Database 'UKIDSSDR4PLUS' cannot be opened due to inaccessible files or insufficient memory or disk space.  See the SQL Server errorlog for details.]
   16.29 +
    17.1 --- a/docker/apache/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    17.2 +++ b/docker/apache/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    17.3 @@ -17,7 +17,7 @@
    17.4  #
    17.5  # Based on example from the official httpd Docker container.
    17.6  # https://github.com/docker-library/httpd/blob/master/2.4/Dockerfile
    17.7 -FROM firethorn/fedora:2.1.31
    17.8 +FROM firethorn/fedora:2.1.33
    17.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   17.10  
   17.11  #
    18.1 --- a/docker/builder/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    18.2 +++ b/docker/builder/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    18.3 @@ -15,7 +15,7 @@
    18.4  # You should have received a copy of the GNU General Public License
    18.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    18.6  
    18.7 -FROM firethorn/java:2.1.31
    18.8 +FROM firethorn/java:2.1.33
    18.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   18.10  
   18.11  #
    19.1 --- a/docker/compose/client/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    19.2 +++ b/docker/compose/client/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    19.3 @@ -15,7 +15,7 @@
    19.4  # You should have received a copy of the GNU General Public License
    19.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    19.6  
    19.7 -FROM firethorn/tester:2.1.31
    19.8 +FROM firethorn/tester:2.1.33
    19.9  LABEL MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   19.10  
   19.11  ENV pysrc=/home/firethorn-py
    20.1 --- a/docker/compose/tests/distictella/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    20.2 +++ b/docker/compose/tests/distictella/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    20.3 @@ -15,7 +15,7 @@
    20.4  # You should have received a copy of the GNU General Public License
    20.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    20.6  
    20.7 -FROM firethorn/fedora:2.1.31
    20.8 +FROM firethorn/fedora:2.1.33
    20.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   20.10  
   20.11  RUN dnf -y install \
    21.1 --- a/docker/compose/tests/distictella/firethorn-py/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    21.2 +++ b/docker/compose/tests/distictella/firethorn-py/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    21.3 @@ -15,7 +15,7 @@
    21.4  # You should have received a copy of the GNU General Public License
    21.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    21.6  
    21.7 -FROM firethorn/fedora:2.1.31
    21.8 +FROM firethorn/fedora:2.1.33
    21.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   21.10  
   21.11  RUN dnf -y install \
    22.1 --- a/docker/fedora/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    22.2 +++ b/docker/fedora/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    22.3 @@ -15,9 +15,14 @@
    22.4  # You should have received a copy of the GNU General Public License
    22.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    22.6  
    22.7 +# FROM latest (fedora:29)
    22.8 +# https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedoralatest
    22.9 +# FROM fedora@sha256:2a60898a6dd7da9964b0c59fedcf652e24bfff04142e5488f793c9e8156afd33
   22.10 +
   22.11 +# Problems with OGSA-DAI webapp meant we had to revert back to fedora:28.
   22.12  # FROM fedora:28
   22.13 -# https://github.com/docker-library/repo-info/blob/master/repos/fedora/tag-details.md#fedora28
   22.14  FROM fedora@sha256:7ae08e5637170eb47c01e315b6e64e0d48c6200d2942c695d0bee61b38c65b39
   22.15 +
   22.16  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
   22.17  
   22.18  #
   22.19 @@ -35,6 +40,7 @@
   22.20  #
   22.21  # Install common admin tools.
   22.22  RUN dnf install -y \
   22.23 +    findutils  \
   22.24      gnupg \
   22.25      gzip  \
   22.26      less  \
   22.27 @@ -47,7 +53,7 @@
   22.28      zip
   22.29  
   22.30  #
   22.31 -# Add our install scripts. 
   22.32 +# Add our install scripts.
   22.33  COPY bin /usr/local/bin/
   22.34  RUN chmod a+x /usr/local/bin/*.sh
   22.35  
    23.1 --- a/docker/java/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    23.2 +++ b/docker/java/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    23.3 @@ -15,7 +15,7 @@
    23.4  # You should have received a copy of the GNU General Public License
    23.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    23.6  
    23.7 -FROM firethorn/fedora:2.1.31
    23.8 +FROM firethorn/fedora:2.1.33
    23.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   23.10  
   23.11  ENV JAVA_VERSION=12.0.0.33
    24.1 --- a/docker/postgres/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    24.2 +++ b/docker/postgres/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    24.3 @@ -15,7 +15,7 @@
    24.4  # You should have received a copy of the GNU General Public License
    24.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    24.6  
    24.7 -FROM postgres:10
    24.8 +FROM postgres:11
    24.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   24.10  
   24.11  #
    25.1 --- a/docker/python/3.4.2/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    25.2 +++ b/docker/python/3.4.2/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    25.3 @@ -1,4 +1,4 @@
    25.4 -FROM firethorn/ubuntu:2.1.31
    25.5 +FROM firethorn/ubuntu:2.1.33
    25.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    25.7  
    25.8  #
    26.1 --- a/docker/pythonlibs/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    26.2 +++ b/docker/pythonlibs/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    26.3 @@ -1,4 +1,4 @@
    26.4 -FROM firethorn/python:2.1.31
    26.5 +FROM firethorn/python:2.1.33
    26.6  MAINTAINER Stelios Voutsinas <stv@roe.ac.uk>
    26.7  
    26.8  # Install Python Libs
    27.1 --- a/docker/sql-proxy/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    27.2 +++ b/docker/sql-proxy/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    27.3 @@ -15,7 +15,7 @@
    27.4  # You should have received a copy of the GNU General Public License
    27.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    27.6  
    27.7 -FROM firethorn/fedora:2.1.31
    27.8 +FROM firethorn/fedora:2.1.33
    27.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   27.10  
   27.11  EXPOSE 1433
    28.1 --- a/docker/sql-tunnel/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    28.2 +++ b/docker/sql-tunnel/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    28.3 @@ -15,7 +15,7 @@
    28.4  # You should have received a copy of the GNU General Public License
    28.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    28.6  
    28.7 -FROM firethorn/fedora:2.1.31
    28.8 +FROM firethorn/fedora:2.1.33
    28.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   28.10  
   28.11  EXPOSE 1433
    29.1 --- a/docker/ssh-client/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    29.2 +++ b/docker/ssh-client/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    29.3 @@ -15,7 +15,7 @@
    29.4  # You should have received a copy of the GNU General Public License
    29.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    29.6  
    29.7 -FROM firethorn/fedora:2.1.31
    29.8 +FROM firethorn/fedora:2.1.33
    29.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   29.10  
   29.11  RUN dnf -y install openssh-clients
    30.1 --- a/docker/tester/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    30.2 +++ b/docker/tester/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    30.3 @@ -15,7 +15,7 @@
    30.4  # You should have received a copy of the GNU General Public License
    30.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    30.6  
    30.7 -FROM firethorn/fedora:2.1.31
    30.8 +FROM firethorn/fedora:2.1.33
    30.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   30.10  
   30.11  RUN dnf install -y jq
    31.1 --- a/docker/tomcat/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    31.2 +++ b/docker/tomcat/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    31.3 @@ -15,11 +15,10 @@
    31.4  # You should have received a copy of the GNU General Public License
    31.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    31.6  
    31.7 -FROM firethorn/java:2.1.31
    31.8 +FROM firethorn/java:2.1.33
    31.9  MAINTAINER [Dave Morris <docker-admin@metagrid.co.uk>]
   31.10  
   31.11  # https://tomcat.apache.org/whichversion.html
   31.12 -#NV TOMCAT_VERSION 8.5.32
   31.13  ENV TOMCAT_VERSION 9.0.10
   31.14  ENV TOMCAT_MAJOR  9
   31.15  
    32.1 --- a/firethorn-admin/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    32.2 +++ b/firethorn-admin/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    32.3 @@ -32,7 +32,7 @@
    32.4      <parent>
    32.5          <groupId>uk.ac.roe.wfau</groupId>
    32.6          <artifactId>firethorn</artifactId>
    32.7 -        <version project='firethorn'>2.1.31</version>
    32.8 +        <version project='firethorn'>2.1.33</version>
    32.9      </parent>
   32.10  
   32.11      <artifactId>firethorn-admin-webapp</artifactId>
    33.1 --- a/firethorn-admin/src/main/docker/firethorn/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    33.2 +++ b/firethorn-admin/src/main/docker/firethorn/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    33.3 @@ -15,7 +15,7 @@
    33.4  # You should have received a copy of the GNU General Public License
    33.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    33.6  
    33.7 -FROM firethorn/tomcat:2.1.31
    33.8 +FROM firethorn/tomcat:2.1.33
    33.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   33.10  
   33.11  #
    34.1 --- a/firethorn-annotations/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    34.2 +++ b/firethorn-annotations/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    34.3 @@ -32,7 +32,7 @@
    34.4      <parent>
    34.5          <groupId>uk.ac.roe.wfau</groupId>
    34.6          <artifactId>firethorn</artifactId>
    34.7 -        <version project='firethorn'>2.1.31</version>
    34.8 +        <version project='firethorn'>2.1.33</version>
    34.9      </parent>
   34.10  
   34.11      <artifactId>firethorn-annotations</artifactId>
    35.1 --- a/firethorn-cds/adql/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    35.2 +++ b/firethorn-cds/adql/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    35.3 @@ -32,7 +32,7 @@
    35.4      <parent>
    35.5          <groupId>uk.ac.roe.wfau</groupId>
    35.6          <artifactId>firethorn-cds</artifactId>
    35.7 -		<version project='firethorn'>2.1.31</version>
    35.8 +		<version project='firethorn'>2.1.33</version>
    35.9  	</parent>
   35.10  
   35.11      <artifactId>firethorn-cds-adql</artifactId>
    36.1 --- a/firethorn-cds/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    36.2 +++ b/firethorn-cds/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    36.3 @@ -32,7 +32,7 @@
    36.4      <parent>
    36.5          <groupId>uk.ac.roe.wfau</groupId>
    36.6          <artifactId>firethorn</artifactId>
    36.7 -        <version project='firethorn'>2.1.31</version>
    36.8 +        <version project='firethorn'>2.1.33</version>
    36.9      </parent>
   36.10  
   36.11      <artifactId>firethorn-cds</artifactId>
    37.1 --- a/firethorn-core/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    37.2 +++ b/firethorn-core/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    37.3 @@ -32,7 +32,7 @@
    37.4      <parent>
    37.5          <groupId>uk.ac.roe.wfau</groupId>
    37.6          <artifactId>firethorn</artifactId>
    37.7 -        <version project='firethorn'>2.1.31</version>
    37.8 +        <version project='firethorn'>2.1.33</version>
    37.9  	</parent>
   37.10  
   37.11      <artifactId>firethorn-core</artifactId>
    38.1 --- a/firethorn-jdbc/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    38.2 +++ b/firethorn-jdbc/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    38.3 @@ -32,7 +32,7 @@
    38.4      <parent>
    38.5          <groupId>uk.ac.roe.wfau</groupId>
    38.6          <artifactId>firethorn</artifactId>
    38.7 -        <version project='firethorn'>2.1.31</version>
    38.8 +        <version project='firethorn'>2.1.33</version>
    38.9      </parent>
   38.10  
   38.11      <artifactId>firethorn-jdbc</artifactId>
   38.12 @@ -77,6 +77,7 @@
   38.13          <dependency>
   38.14              <groupId>org.hsqldb</groupId>
   38.15              <artifactId>hsqldb</artifactId>
   38.16 +            <!--version>2.5.0</version-->
   38.17              <version>2.4.1</version>
   38.18          </dependency>
   38.19  
   38.20 @@ -89,6 +90,7 @@
   38.21      	<dependency>
   38.22      		<groupId>org.postgresql</groupId>
   38.23      		<artifactId>postgresql</artifactId>
   38.24 +            <!--version>42.2.5</version-->
   38.25              <version>42.2.4</version>
   38.26      	</dependency>
   38.27  
   38.28 @@ -100,6 +102,7 @@
   38.29          <dependency>
   38.30              <groupId>mysql</groupId>
   38.31              <artifactId>mysql-connector-java</artifactId>
   38.32 +            <!--version>8.0.16</version-->
   38.33              <version>8.0.12</version>
   38.34          </dependency>
   38.35  
   38.36 @@ -110,6 +113,7 @@
   38.37          <dependency>
   38.38              <groupId>org.mariadb.jdbc</groupId>
   38.39              <artifactId>mariadb-java-client</artifactId>
   38.40 +            <!--version>2.4.1</version-->
   38.41              <version>2.2.6</version>
   38.42          </dependency>
   38.43  
    39.1 --- a/firethorn-logging/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    39.2 +++ b/firethorn-logging/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    39.3 @@ -32,7 +32,7 @@
    39.4      <parent>
    39.5          <groupId>uk.ac.roe.wfau</groupId>
    39.6          <artifactId>firethorn</artifactId>
    39.7 -        <version project='firethorn'>2.1.31</version>
    39.8 +        <version project='firethorn'>2.1.33</version>
    39.9      </parent>
   39.10  
   39.11      <artifactId>firethorn-logging</artifactId>
   39.12 @@ -61,7 +61,9 @@
   39.13  
   39.14          <logtype>logback</logtype>
   39.15  
   39.16 +		<!--org.slf4j.version>1.7.26</org.slf4j.version-->
   39.17  		<org.slf4j.version>1.7.25</org.slf4j.version>
   39.18 +
   39.19          <org.log4j.version>1.2.17</org.log4j.version>
   39.20          <org.logback.version>1.2.3</org.logback.version>
   39.21          <commons.logging.version>1.2</commons.logging.version>
    40.1 --- a/firethorn-ogsadai/activity/client/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    40.2 +++ b/firethorn-ogsadai/activity/client/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    40.3 @@ -32,7 +32,7 @@
    40.4      <parent>
    40.5          <groupId>uk.ac.roe.wfau</groupId>
    40.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    40.7 -        <version project='firethorn'>2.1.31</version>
    40.8 +        <version project='firethorn'>2.1.33</version>
    40.9  	</parent>
   40.10  
   40.11      <artifactId>firethorn-ogsadai-activity-client</artifactId>
    41.1 --- a/firethorn-ogsadai/activity/common/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    41.2 +++ b/firethorn-ogsadai/activity/common/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    41.3 @@ -32,7 +32,7 @@
    41.4      <parent>
    41.5          <groupId>uk.ac.roe.wfau</groupId>
    41.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    41.7 -        <version project='firethorn'>2.1.31</version>
    41.8 +        <version project='firethorn'>2.1.33</version>
    41.9      </parent>
   41.10  
   41.11      <artifactId>firethorn-ogsadai-activity-common</artifactId>
    42.1 --- a/firethorn-ogsadai/activity/context/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    42.2 +++ b/firethorn-ogsadai/activity/context/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    42.3 @@ -32,7 +32,7 @@
    42.4      <parent>
    42.5          <groupId>uk.ac.roe.wfau</groupId>
    42.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    42.7 -        <version project='firethorn'>2.1.31</version>
    42.8 +        <version project='firethorn'>2.1.33</version>
    42.9  	</parent>
   42.10  
   42.11      <artifactId>firethorn-ogsadai-activity-context</artifactId>
    43.1 --- a/firethorn-ogsadai/activity/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    43.2 +++ b/firethorn-ogsadai/activity/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    43.3 @@ -37,7 +37,7 @@
    43.4      <parent>
    43.5          <groupId>uk.ac.roe.wfau</groupId>
    43.6          <artifactId>firethorn-ogsadai</artifactId>
    43.7 -        <version project='firethorn'>2.1.31</version>
    43.8 +        <version project='firethorn'>2.1.33</version>
    43.9      </parent>
   43.10  
   43.11      <artifactId>firethorn-ogsadai-activity</artifactId>
    44.1 --- a/firethorn-ogsadai/activity/server/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    44.2 +++ b/firethorn-ogsadai/activity/server/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    44.3 @@ -32,7 +32,7 @@
    44.4      <parent>
    44.5          <groupId>uk.ac.roe.wfau</groupId>
    44.6          <artifactId>firethorn-ogsadai-activity</artifactId>
    44.7 -        <version project='firethorn'>2.1.31</version>
    44.8 +        <version project='firethorn'>2.1.33</version>
    44.9      </parent>
   44.10  
   44.11      <artifactId>firethorn-ogsadai-activity-server</artifactId>
    45.1 --- a/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java	Tue Jul 02 04:12:16 2019 +0200
    45.2 +++ b/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java	Tue Jul 02 04:12:49 2019 +0200
    45.3 @@ -478,21 +478,45 @@
    45.4      protected void cleanUp() throws Exception
    45.5      {
    45.6          super.cleanUp();
    45.7 -        
    45.8 -        logger.debug("Shutting down ExecutorService");
    45.9 -        mExecutorService.shutdown();
   45.10 -        
   45.11 +
   45.12 +        logger.debug("cleanUp()");
   45.13 +        if (mExecutorService != null)
   45.14 +        {
   45.15 +            try {
   45.16 +                logger.debug("Shutting down ExecutorService");
   45.17 +                mContext.monkey().sqlException(this, "caiR6thi");
   45.18 +                mExecutorService.shutdown();
   45.19 +            }
   45.20 +            catch (final Exception ouch)
   45.21 +            {
   45.22 +                logger.debug("Shutting down ExecutorService threw an Exception [" + ouch.getMessage() + "]");
   45.23 +            }
   45.24 +        }
   45.25          if (mStatement != null)
   45.26          {
   45.27 -            logger.debug("Closing Statement");
   45.28 -            mContext.monkey().sqlException(this, "baivahP0");
   45.29 -            mStatement.close();
   45.30 +            try {
   45.31 +                logger.debug("Closing Statement");
   45.32 +                mContext.monkey().sqlException(this, "baivahP0");
   45.33 +                mStatement.close();
   45.34 +            }
   45.35 +            catch (final Exception ouch)
   45.36 +            {
   45.37 +                logger.debug("Closing Statement threw an Exception [" + ouch.getMessage() + "]");
   45.38 +            }
   45.39          }
   45.40  
   45.41 -        if (mResource != null)
   45.42 +        if (mConnection != null)
   45.43          {
   45.44 -            logger.debug("Releasing Connection");
   45.45 -            mConnectionProvider.releaseConnection(mConnection);
   45.46 +            try {
   45.47 +                logger.debug("Releasing Connection");
   45.48 +                mContext.monkey().sqlException(this, "echo4ieW");
   45.49 +                mConnectionProvider.releaseConnection(mConnection);
   45.50 +            }
   45.51 +            catch (final Exception ouch)
   45.52 +            {
   45.53 +                logger.debug("Releasing Connection threw an Exception [" + ouch.getMessage() + "]");
   45.54 +            }
   45.55          }
   45.56      }
   45.57 +
   45.58  }
    46.1 --- a/firethorn-ogsadai/astro/client/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    46.2 +++ b/firethorn-ogsadai/astro/client/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    46.3 @@ -32,7 +32,7 @@
    46.4      <parent>
    46.5          <groupId>uk.ac.roe.wfau</groupId>
    46.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    46.7 -        <version project='firethorn'>2.1.31</version>
    46.8 +        <version project='firethorn'>2.1.33</version>
    46.9      </parent>
   46.10  
   46.11      <artifactId>firethorn-ogsadai-astro-client</artifactId>
    47.1 --- a/firethorn-ogsadai/astro/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    47.2 +++ b/firethorn-ogsadai/astro/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    47.3 @@ -32,7 +32,7 @@
    47.4      <parent>
    47.5          <groupId>uk.ac.roe.wfau</groupId>
    47.6          <artifactId>firethorn-ogsadai</artifactId>
    47.7 -        <version project='firethorn'>2.1.31</version>
    47.8 +        <version project='firethorn'>2.1.33</version>
    47.9      </parent>
   47.10  
   47.11      <artifactId>firethorn-ogsadai-astro</artifactId>
    48.1 --- a/firethorn-ogsadai/astro/server/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    48.2 +++ b/firethorn-ogsadai/astro/server/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    48.3 @@ -32,7 +32,7 @@
    48.4      <parent>
    48.5          <groupId>uk.ac.roe.wfau</groupId>
    48.6          <artifactId>firethorn-ogsadai-astro</artifactId>
    48.7 -        <version project='firethorn'>2.1.31</version>
    48.8 +        <version project='firethorn'>2.1.33</version>
    48.9      </parent>
   48.10  
   48.11      <artifactId>firethorn-ogsadai-astro-server</artifactId>
    49.1 --- a/firethorn-ogsadai/depends/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    49.2 +++ b/firethorn-ogsadai/depends/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    49.3 @@ -32,7 +32,7 @@
    49.4      <parent>
    49.5          <groupId>uk.ac.roe.wfau</groupId>
    49.6          <artifactId>firethorn-ogsadai</artifactId>
    49.7 -        <version project='firethorn'>2.1.31</version>
    49.8 +        <version project='firethorn'>2.1.33</version>
    49.9      </parent>
   49.10  
   49.11      <artifactId>firethorn-ogsadai-depends</artifactId>
    50.1 --- a/firethorn-ogsadai/distributed/client/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    50.2 +++ b/firethorn-ogsadai/distributed/client/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    50.3 @@ -32,7 +32,7 @@
    50.4      <parent>
    50.5          <groupId>uk.ac.roe.wfau</groupId>
    50.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    50.7 -        <version project='firethorn'>2.1.31</version>
    50.8 +        <version project='firethorn'>2.1.33</version>
    50.9      </parent>
   50.10  
   50.11      <artifactId>firethorn-ogsadai-dqp-client</artifactId>
    51.1 --- a/firethorn-ogsadai/distributed/common/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    51.2 +++ b/firethorn-ogsadai/distributed/common/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    51.3 @@ -32,7 +32,7 @@
    51.4      <parent>
    51.5          <groupId>uk.ac.roe.wfau</groupId>
    51.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    51.7 -        <version project='firethorn'>2.1.31</version>
    51.8 +        <version project='firethorn'>2.1.33</version>
    51.9      </parent>
   51.10  
   51.11      <artifactId>firethorn-ogsadai-dqp-common</artifactId>
    52.1 --- a/firethorn-ogsadai/distributed/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    52.2 +++ b/firethorn-ogsadai/distributed/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    52.3 @@ -32,7 +32,7 @@
    52.4      <parent>
    52.5          <groupId>uk.ac.roe.wfau</groupId>
    52.6          <artifactId>firethorn-ogsadai</artifactId>
    52.7 -        <version project='firethorn'>2.1.31</version>
    52.8 +        <version project='firethorn'>2.1.33</version>
    52.9      </parent>
   52.10  
   52.11      <artifactId>firethorn-ogsadai-dqp</artifactId>
    53.1 --- a/firethorn-ogsadai/distributed/server/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    53.2 +++ b/firethorn-ogsadai/distributed/server/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    53.3 @@ -32,7 +32,7 @@
    53.4      <parent>
    53.5          <groupId>uk.ac.roe.wfau</groupId>
    53.6          <artifactId>firethorn-ogsadai-dqp</artifactId>
    53.7 -        <version project='firethorn'>2.1.31</version>
    53.8 +        <version project='firethorn'>2.1.33</version>
    53.9      </parent>
   53.10  
   53.11      <artifactId>firethorn-ogsadai-dqp-server</artifactId>
    54.1 --- a/firethorn-ogsadai/firethorn/client/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    54.2 +++ b/firethorn-ogsadai/firethorn/client/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    54.3 @@ -32,7 +32,7 @@
    54.4      <parent>
    54.5          <groupId>uk.ac.roe.wfau</groupId>
    54.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    54.7 -        <version project='firethorn'>2.1.31</version>
    54.8 +        <version project='firethorn'>2.1.33</version>
    54.9      </parent>
   54.10  
   54.11      <artifactId>firethorn-ogsadai-firethorn-client</artifactId>
    55.1 --- a/firethorn-ogsadai/firethorn/common/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    55.2 +++ b/firethorn-ogsadai/firethorn/common/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    55.3 @@ -32,7 +32,7 @@
    55.4      <parent>
    55.5          <groupId>uk.ac.roe.wfau</groupId>
    55.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    55.7 -        <version project='firethorn'>2.1.31</version>
    55.8 +        <version project='firethorn'>2.1.33</version>
    55.9      </parent>
   55.10  
   55.11      <artifactId>firethorn-ogsadai-firethorn-common</artifactId>
    56.1 --- a/firethorn-ogsadai/firethorn/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    56.2 +++ b/firethorn-ogsadai/firethorn/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    56.3 @@ -32,7 +32,7 @@
    56.4      <parent>
    56.5          <groupId>uk.ac.roe.wfau</groupId>
    56.6          <artifactId>firethorn-ogsadai</artifactId>
    56.7 -        <version project='firethorn'>2.1.31</version>
    56.8 +        <version project='firethorn'>2.1.33</version>
    56.9      </parent>
   56.10  
   56.11      <artifactId>firethorn-ogsadai-firethorn</artifactId>
    57.1 --- a/firethorn-ogsadai/firethorn/server/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    57.2 +++ b/firethorn-ogsadai/firethorn/server/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    57.3 @@ -32,7 +32,7 @@
    57.4      <parent>
    57.5          <groupId>uk.ac.roe.wfau</groupId>
    57.6          <artifactId>firethorn-ogsadai-firethorn</artifactId>
    57.7 -        <version project='firethorn'>2.1.31</version>
    57.8 +        <version project='firethorn'>2.1.33</version>
    57.9      </parent>
   57.10  
   57.11      <artifactId>firethorn-ogsadai-firethorn-server</artifactId>
    58.1 --- a/firethorn-ogsadai/metadata/client/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    58.2 +++ b/firethorn-ogsadai/metadata/client/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    58.3 @@ -32,7 +32,7 @@
    58.4      <parent>
    58.5          <groupId>uk.ac.roe.wfau</groupId>
    58.6          <artifactId>firethorn-ogsadai-metadata</artifactId>
    58.7 -        <version project='firethorn'>2.1.31</version>
    58.8 +        <version project='firethorn'>2.1.33</version>
    58.9      </parent>
   58.10  
   58.11      <artifactId>firethorn-ogsadai-metadata-client</artifactId>
    59.1 --- a/firethorn-ogsadai/metadata/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    59.2 +++ b/firethorn-ogsadai/metadata/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    59.3 @@ -32,7 +32,7 @@
    59.4      <parent>
    59.5          <groupId>uk.ac.roe.wfau</groupId>
    59.6          <artifactId>firethorn-ogsadai</artifactId>
    59.7 -        <version project='firethorn'>2.1.31</version>
    59.8 +        <version project='firethorn'>2.1.33</version>
    59.9      </parent>
   59.10  
   59.11      <artifactId>firethorn-ogsadai-metadata</artifactId>
    60.1 --- a/firethorn-ogsadai/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    60.2 +++ b/firethorn-ogsadai/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    60.3 @@ -37,7 +37,7 @@
    60.4      <parent>
    60.5          <groupId>uk.ac.roe.wfau</groupId>
    60.6          <artifactId>firethorn</artifactId>
    60.7 -        <version project='firethorn'>2.1.31</version>
    60.8 +        <version project='firethorn'>2.1.33</version>
    60.9      </parent>
   60.10  
   60.11      <artifactId>firethorn-ogsadai</artifactId>
    61.1 --- a/firethorn-ogsadai/webapp/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    61.2 +++ b/firethorn-ogsadai/webapp/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    61.3 @@ -32,7 +32,7 @@
    61.4      <parent>
    61.5          <groupId>uk.ac.roe.wfau</groupId>
    61.6          <artifactId>firethorn-ogsadai</artifactId>
    61.7 -        <version project='firethorn'>2.1.31</version>
    61.8 +        <version project='firethorn'>2.1.33</version>
    61.9      </parent>
   61.10  
   61.11      <artifactId>firethorn-ogsadai-webapp</artifactId>
    62.1 --- a/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    62.2 +++ b/firethorn-ogsadai/webapp/src/main/docker/ogsadai/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    62.3 @@ -15,7 +15,7 @@
    62.4  # You should have received a copy of the GNU General Public License
    62.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    62.6  
    62.7 -FROM firethorn/tomcat:2.1.31
    62.8 +FROM firethorn/tomcat:2.1.33
    62.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   62.10  
   62.11  #
    63.1 --- a/firethorn-spring/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    63.2 +++ b/firethorn-spring/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    63.3 @@ -32,7 +32,7 @@
    63.4      <parent>
    63.5          <groupId>uk.ac.roe.wfau</groupId>
    63.6          <artifactId>firethorn</artifactId>
    63.7 -        <version project='firethorn'>2.1.31</version>
    63.8 +        <version project='firethorn'>2.1.33</version>
    63.9      </parent>
   63.10  
   63.11      <artifactId>firethorn-spring</artifactId>
   63.12 @@ -49,11 +49,19 @@
   63.13  
   63.14      <properties>
   63.15  
   63.16 +		<!--spring.version>5.1.7.RELEASE</spring.version-->
   63.17  		<spring.version>5.0.8.RELEASE</spring.version>
   63.18 +
   63.19 +        <!--hibernate.version>5.4.3.Final</hibernate.version-->
   63.20          <hibernate.version>5.3.1.Final</hibernate.version>
   63.21 +
   63.22 +	    <!--hibernate.validator>6.0.16.Final</hibernate.validator-->
   63.23  	    <hibernate.validator>6.0.11.Final</hibernate.validator>
   63.24 +
   63.25 +		<!--aspectj.version>1.9.4</aspectj.version-->
   63.26  		<aspectj.version>1.9.1</aspectj.version>
   63.27  
   63.28 +        <!--jackson.version>2.9.9</jackson.version-->
   63.29          <jackson.version>2.9.6</jackson.version>
   63.30  
   63.31      </properties>
   63.32 @@ -147,6 +155,7 @@
   63.33          <dependency>
   63.34              <groupId>cglib</groupId>
   63.35              <artifactId>cglib</artifactId>
   63.36 +            <!--version>3.2.12</version-->
   63.37              <version>3.2.7</version>
   63.38          </dependency>
   63.39  
    64.1 --- a/firethorn-stil/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    64.2 +++ b/firethorn-stil/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    64.3 @@ -32,7 +32,7 @@
    64.4      <parent>
    64.5          <groupId>uk.ac.roe.wfau</groupId>
    64.6          <artifactId>firethorn</artifactId>
    64.7 -        <version project='firethorn'>2.1.31</version>
    64.8 +        <version project='firethorn'>2.1.33</version>
    64.9      </parent>
   64.10  
   64.11      <artifactId>firethorn-stil</artifactId>
    65.1 --- a/firethorn-testing/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    65.2 +++ b/firethorn-testing/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    65.3 @@ -32,7 +32,7 @@
    65.4      <parent>
    65.5          <groupId>uk.ac.roe.wfau</groupId>
    65.6          <artifactId>firethorn</artifactId>
    65.7 -        <version project='firethorn'>2.1.31</version>
    65.8 +        <version project='firethorn'>2.1.33</version>
    65.9      </parent>
   65.10  
   65.11      <artifactId>firethorn-testing</artifactId>
    66.1 --- a/firethorn-webapp/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    66.2 +++ b/firethorn-webapp/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    66.3 @@ -32,7 +32,7 @@
    66.4      <parent>
    66.5          <groupId>uk.ac.roe.wfau</groupId>
    66.6          <artifactId>firethorn</artifactId>
    66.7 -        <version project='firethorn'>2.1.31</version>
    66.8 +        <version project='firethorn'>2.1.33</version>
    66.9      </parent>
   66.10  
   66.11      <artifactId>firethorn-webapp</artifactId>
   66.12 @@ -200,7 +200,7 @@
   66.13              <plugin>
   66.14                  <groupId>org.apache.maven.plugins</groupId>
   66.15                  <artifactId>maven-project-info-reports-plugin</artifactId>
   66.16 -                <version>2.9</version>
   66.17 +                <version>3.0.0</version>
   66.18                  <reportSets>
   66.19                      <reportSet>
   66.20                          <reports>
    67.1 --- a/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    67.2 +++ b/firethorn-webapp/src/main/docker/firethorn/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    67.3 @@ -15,7 +15,7 @@
    67.4  # You should have received a copy of the GNU General Public License
    67.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    67.6  
    67.7 -FROM firethorn/tomcat:2.1.31
    67.8 +FROM firethorn/tomcat:2.1.33
    67.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   67.10  
   67.11  #
    68.1 --- a/firethorn-webapp/src/main/docker/firethorn/conf/server.xml	Tue Jul 02 04:12:16 2019 +0200
    68.2 +++ b/firethorn-webapp/src/main/docker/firethorn/conf/server.xml	Tue Jul 02 04:12:49 2019 +0200
    68.3 @@ -62,7 +62,7 @@
    68.4      <Executor
    68.5          name="main-interface"
    68.6          namePrefix="main-interface-"
    68.7 -        maxThreads="20"
    68.8 +        maxThreads="10"
    68.9          minSpareThreads="5"
   68.10          />
   68.11  
    69.1 --- a/integration/005/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    69.2 +++ b/integration/005/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    69.3 @@ -32,7 +32,7 @@
    69.4      <parent>
    69.5          <groupId>uk.ac.roe.wfau</groupId>
    69.6          <artifactId>firethorn</artifactId>
    69.7 -        <version project='firethorn'>2.1.31</version>
    69.8 +        <version project='firethorn'>2.1.33</version>
    69.9      </parent>
   69.10  
   69.11      <artifactId>firethorn-integration-tests</artifactId>
    70.1 --- a/integration/tester/Dockerfile	Tue Jul 02 04:12:16 2019 +0200
    70.2 +++ b/integration/tester/Dockerfile	Tue Jul 02 04:12:49 2019 +0200
    70.3 @@ -15,7 +15,7 @@
    70.4  # You should have received a copy of the GNU General Public License
    70.5  # along with this program.  If not, see <http://www.gnu.org/licenses/>.
    70.6  
    70.7 -FROM firethorn/fedora:2.1.31
    70.8 +FROM firethorn/fedora:2.1.33
    70.9  MAINTAINER Dave Morris <docker-admin@metagrid.co.uk>
   70.10  
   70.11  RUN dnf -y install perl
    71.1 --- a/pom.xml	Tue Jul 02 04:12:16 2019 +0200
    71.2 +++ b/pom.xml	Tue Jul 02 04:12:49 2019 +0200
    71.3 @@ -39,7 +39,7 @@
    71.4          | Maven project versions
    71.5          | http://www.sonatype.com/books/mvnref-book/reference/pom-relationships-sect-pom-syntax.html
    71.6          +-->
    71.7 -    <version project='firethorn'>2.1.31</version>
    71.8 +    <version project='firethorn'>2.1.33</version>
    71.9  
   71.10      <prerequisites>
   71.11          <maven>3.3.1</maven>
   71.12 @@ -295,6 +295,7 @@
   71.13              <dependency>
   71.14                  <groupId>commons-codec</groupId>
   71.15                  <artifactId>commons-codec</artifactId>
   71.16 +                <!--version>1.12</version-->
   71.17                  <version>1.11</version>
   71.18              </dependency>
   71.19              <dependency>
   71.20 @@ -315,12 +316,13 @@
   71.21              <dependency>
   71.22                  <groupId>org.apache.commons</groupId>
   71.23                  <artifactId>commons-lang3</artifactId>
   71.24 +                <!--version>3.9</version-->
   71.25                  <version>3.7</version>
   71.26              </dependency>
   71.27          	<dependency>
   71.28                  <groupId>commons-fileupload</groupId>
   71.29                  <artifactId>commons-fileupload</artifactId>
   71.30 -                <version>1.3.3</version>
   71.31 +                <version>1.4</version>
   71.32          	</dependency>
   71.33              <dependency>
   71.34                  <groupId>commons-io</groupId>
   71.35 @@ -334,6 +336,7 @@
   71.36              <dependency>
   71.37                  <groupId>org.apache.httpcomponents</groupId>
   71.38                  <artifactId>httpclient</artifactId>
   71.39 +                <!--version>4.5.8</version-->
   71.40                  <version>4.5.6</version>
   71.41              </dependency>
   71.42  
   71.43 @@ -344,6 +347,7 @@
   71.44              <dependency>
   71.45                  <groupId>com.google.guava</groupId>
   71.46                  <artifactId>guava</artifactId>
   71.47 +                <!--version>27.1-jre</version-->
   71.48                  <version>26.0-jre</version>
   71.49              </dependency>
   71.50  
   71.51 @@ -393,6 +397,7 @@
   71.52              <dependency>
   71.53                  <groupId>joda-time</groupId>
   71.54                  <artifactId>joda-time</artifactId>
   71.55 +                <!--version>2.12</version-->
   71.56                  <version>2.10</version>
   71.57              </dependency>
   71.58              <dependency>
   71.59 @@ -435,6 +440,7 @@
   71.60              <dependency>
   71.61                  <groupId>org.jasypt</groupId>
   71.62                  <artifactId>jasypt</artifactId>
   71.63 +                <!--version>1.9.3</version-->
   71.64                  <version>1.9.2</version>
   71.65              </dependency>
   71.66  
   71.67 @@ -460,6 +466,7 @@
   71.68              <dependency>
   71.69                  <groupId>org.projectlombok</groupId>
   71.70                  <artifactId>lombok</artifactId>
   71.71 +                <!--version>1.18.8</version-->
   71.72                  <version>1.18.2</version>
   71.73              </dependency>
   71.74  
   71.75 @@ -524,7 +531,7 @@
   71.76  
   71.77                  <plugin>
   71.78                      <artifactId>maven-compiler-plugin</artifactId>
   71.79 -                    <version>3.7.0</version>
   71.80 +                    <version>3.8.1</version>
   71.81                      <configuration>
   71.82                          <source>${project.javaversion}</source>
   71.83                          <target>${project.javaversion}</target>
   71.84 @@ -573,17 +580,17 @@
   71.85                  <plugin>
   71.86                      <groupId>org.apache.maven.plugins</groupId>
   71.87                      <artifactId>maven-jar-plugin</artifactId>
   71.88 -                    <version>3.1.0</version>
   71.89 +                    <version>3.1.2</version>
   71.90                  </plugin>
   71.91                  <plugin>
   71.92                      <groupId>org.apache.maven.plugins</groupId>
   71.93                      <artifactId>maven-war-plugin</artifactId>
   71.94 -                    <version>3.2.2</version>
   71.95 +                    <version>3.2.3</version>
   71.96                  </plugin>
   71.97                  <plugin>
   71.98                      <groupId>org.apache.maven.plugins</groupId>
   71.99                      <artifactId>maven-source-plugin</artifactId>
  71.100 -                    <version>3.0.1</version>
  71.101 +                    <version>3.1.0</version>
  71.102                  </plugin>
  71.103                  <plugin>
  71.104                      <groupId>org.codehaus.mojo</groupId>
  71.105 @@ -599,7 +606,7 @@
  71.106                  <plugin>
  71.107                      <groupId>org.apache.maven.plugins</groupId>
  71.108                      <artifactId>maven-surefire-plugin</artifactId>
  71.109 -                    <version>2.22.0</version>
  71.110 +                    <version>2.22.2</version>
  71.111                  </plugin>
  71.112  
  71.113                  <plugin>
  71.114 @@ -902,7 +909,7 @@
  71.115              <plugin>
  71.116                  <groupId>org.apache.maven.plugins</groupId>
  71.117                  <artifactId>maven-javadoc-plugin</artifactId>
  71.118 -                <version>3.0.1</version>
  71.119 +                <version>3.1.0</version>
  71.120  
  71.121                  <configuration>
  71.122                      <detectJavaApiLink>true</detectJavaApiLink>
  71.123 @@ -962,7 +969,7 @@
  71.124              <plugin>
  71.125                  <groupId>org.apache.maven.plugins</groupId>
  71.126                  <artifactId>maven-surefire-report-plugin</artifactId>
  71.127 -                <version>2.22.0</version>
  71.128 +                <version>2.22.2</version>
  71.129              </plugin>
  71.130  
  71.131              <!--+
  71.132 @@ -989,7 +996,7 @@
  71.133              <plugin>
  71.134                  <groupId>org.apache.maven.plugins</groupId>
  71.135                  <artifactId>maven-jxr-plugin</artifactId>
  71.136 -                <version>2.5</version>
  71.137 +                <version>3.0.0</version>
  71.138                  <configuration>
  71.139                      <linkJavadoc>true</linkJavadoc>
  71.140                  </configuration>