firethorn

changeset 4241:0464b6306f73

Notes on Hg merge process
author Dave Morris <dmr@roe.ac.uk>
date Mon Jun 03 04:46:00 2019 +0100 (2019-06-03)
parents d581dc232252
children 0c760661dfde e9d24ed64319
files doc/notes/zrq/20190508-01-vmdown-debug.txt doc/notes/zrq/20190508-02-vmdown-debug.txt doc/notes/zrq/20190601-01-hg-merge.txt doc/notes/zrq/20190601-02-merge-tests.txt
line diff
     1.1 --- a/doc/notes/zrq/20190508-01-vmdown-debug.txt	Mon Jun 03 04:24:03 2019 +0100
     1.2 +++ b/doc/notes/zrq/20190508-01-vmdown-debug.txt	Mon Jun 03 04:46:00 2019 +0100
     1.3 @@ -26,18 +26,16 @@
     1.4  
     1.5      docker ps -a
     1.6  
     1.7 ---START--
     1.8 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
     1.9 -16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                12 days ago         Exited (137) 12 days ago                              stevedore_firethorn-py_run_5
    1.10 -2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 12 days ago                              stevedore_firethorn-py_run_4
    1.11 -659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
    1.12 -cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
    1.13 -ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
    1.14 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
    1.15 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   8080/tcp                 stevedore_jarmila_1
    1.16 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_carolina_1
    1.17 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_bethany_1
    1.18 ---END--
    1.19 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
    1.20 +    >   16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                12 days ago         Exited (137) 12 days ago                              stevedore_firethorn-py_run_5
    1.21 +    >   2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 12 days ago                              stevedore_firethorn-py_run_4
    1.22 +    >   659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
    1.23 +    >   cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
    1.24 +    >   ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
    1.25 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
    1.26 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   8080/tcp                 stevedore_jarmila_1
    1.27 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_carolina_1
    1.28 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_bethany_1
    1.29  
    1.30  
    1.31  # -----------------------------------------------------
    1.32 @@ -50,15 +48,13 @@
    1.33  
    1.34      docker-compose --file "docker-compose.yml" run firethorn-py
    1.35  
    1.36 ---START--
    1.37 -Starting stevedore_bethany_1  ... done
    1.38 -Starting stevedore_jarmila_1  ... done
    1.39 -Starting stevedore_carolina_1 ... done
    1.40 -Starting stevedore_gillian_1  ... done
    1.41 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    1.42 -[GCC 5.4.0 20160609] on linux
    1.43 -Type "help", "copyright", "credits" or "license" for more information.
    1.44 ---END--
    1.45 +    >   Starting stevedore_bethany_1  ... done
    1.46 +    >   Starting stevedore_jarmila_1  ... done
    1.47 +    >   Starting stevedore_carolina_1 ... done
    1.48 +    >   Starting stevedore_gillian_1  ... done
    1.49 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    1.50 +    >   [GCC 5.4.0 20160609] on linux
    1.51 +    >   Type "help", "copyright", "credits" or "license" for more information.
    1.52  
    1.53  
    1.54  # -----------------------------------------------------
    1.55 @@ -167,167 +163,165 @@
    1.56  # *lots* of broken :-(
    1.57  #
    1.58  
    1.59 ---START--
    1.60 -{
    1.61 -  "resources": [],
    1.62 -  "owner": "http://gillian:8080/firethorn/community-member/57698559",
    1.63 -  "text": null,
    1.64 -  "callback": "http://gillian:8081/firethorn/callback/57698652",
    1.65 -  "modified": "2019-05-08T19:34:59.347",
    1.66 -  "history": [],
    1.67 -  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
    1.68 -  "delays": {
    1.69 -    "first": null,
    1.70 -    "last": null,
    1.71 -    "every": null
    1.72 -  },
    1.73 -  "url": "http://gillian:8080/firethorn/blue/query/57698652",
    1.74 -  "ident": "57698652",
    1.75 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
    1.76 -  "columns": [],
    1.77 -  "fields": [],
    1.78 -  "tables": [],
    1.79 -  "mode": "AUTO",
    1.80 -  "created": "2019-05-08T19:34:59.210",
    1.81 -  "name": "XX_5U5BMFLXCC4E4AAAAFVJR4I3RI",
    1.82 -  "results": {
    1.83 -    "count": 0,
    1.84 -    "state": "NONE",
    1.85 -    "formats": {
    1.86 -      "datatable": null,
    1.87 -      "votable": null
    1.88 -    },
    1.89 -    "table": null
    1.90 -  },
    1.91 -  "osql": null,
    1.92 -  "adql": null,
    1.93 -  "workspace": "http://gillian:8080/firethorn/adql/resource/57688319",
    1.94 -  "limits": {
    1.95 -    "cells": null,
    1.96 -    "rows": 1000000,
    1.97 -    "time": null
    1.98 -  },
    1.99 -  "resource": "http://gillian:8080/firethorn/adql/resource/57688319",
   1.100 -  "status": "FAILED",
   1.101 -  "syntax": {
   1.102 -    "status": "PARSE_ERROR",
   1.103 -    "friendly": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !",
   1.104 -    "message": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !"
   1.105 -  },
   1.106 -  "self": "http://gillian:8080/firethorn/blue/query/57698652"
   1.107 -}
   1.108 ->>> print(
   1.109 -...     query_obj.table()
   1.110 -...     )
   1.111 -ERROR:root:unknown url type: 'None'
   1.112 -Traceback (most recent call last):
   1.113 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.114 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.115 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.116 -    self.full_url = url
   1.117 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.118 -    self._parse()
   1.119 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.120 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.121 -ValueError: unknown url type: 'None'
   1.122 -ERROR:root:unknown url type: 'None'
   1.123 -Traceback (most recent call last):
   1.124 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.125 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.126 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.127 -    self.full_url = url
   1.128 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.129 -    self._parse()
   1.130 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.131 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.132 -ValueError: unknown url type: 'None'
   1.133 -ERROR:root:unknown url type: 'None'
   1.134 -Traceback (most recent call last):
   1.135 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.136 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.137 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.138 -    self.full_url = url
   1.139 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.140 -    self._parse()
   1.141 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.142 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.143 -ValueError: unknown url type: 'None'
   1.144 -null
   1.145 ->>> print(
   1.146 -...     query_obj.table().count()
   1.147 -...     )
   1.148 -ERROR:root:unknown url type: 'None'
   1.149 -Traceback (most recent call last):
   1.150 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.151 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.152 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.153 -    self.full_url = url
   1.154 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.155 -    self._parse()
   1.156 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.157 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.158 -ValueError: unknown url type: 'None'
   1.159 -ERROR:root:unknown url type: 'None'
   1.160 -Traceback (most recent call last):
   1.161 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.162 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.163 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.164 -    self.full_url = url
   1.165 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.166 -    self._parse()
   1.167 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.168 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.169 -ValueError: unknown url type: 'None'
   1.170 -ERROR:root:unknown url type: 'None'
   1.171 -Traceback (most recent call last):
   1.172 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.173 -    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.174 -  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.175 -    self.full_url = url
   1.176 -  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.177 -    self._parse()
   1.178 -  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.179 -    raise ValueError("unknown url type: %r" % self.full_url)
   1.180 -ValueError: unknown url type: 'None'
   1.181 -None
   1.182 ->>>
   1.183 ->>>
   1.184 ->>>
   1.185 ->>>
   1.186 ->>>
   1.187 ->>> resource = firethorn.firethorn_engine.select_adql_resources()[28]
   1.188 -ERROR:root:IncompleteRead(9713032 bytes read)
   1.189 -Traceback (most recent call last):
   1.190 -  File "/usr/lib/python3.5/http/client.py", line 541, in _get_chunk_left
   1.191 -    chunk_left = self._read_next_chunk_size()
   1.192 -  File "/usr/lib/python3.5/http/client.py", line 508, in _read_next_chunk_size
   1.193 -    return int(line, 16)
   1.194 -ValueError: invalid literal for int() with base 16: b''
   1.195 -
   1.196 -During handling of the above exception, another exception occurred:
   1.197 -
   1.198 -Traceback (most recent call last):
   1.199 -  File "/usr/lib/python3.5/http/client.py", line 558, in _readall_chunked
   1.200 -    chunk_left = self._get_chunk_left()
   1.201 -  File "/usr/lib/python3.5/http/client.py", line 543, in _get_chunk_left
   1.202 -    raise IncompleteRead(b'')
   1.203 -http.client.IncompleteRead: IncompleteRead(0 bytes read)
   1.204 -
   1.205 -During handling of the above exception, another exception occurred:
   1.206 -
   1.207 -Traceback (most recent call last):
   1.208 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/core/firethorn_engine.py", line 238, in select_adql_resources
   1.209 -    adqlresources =  json.loads(response.read().decode('utf-8'))
   1.210 -  File "/usr/lib/python3.5/http/client.py", line 455, in read
   1.211 -    return self._readall_chunked()
   1.212 -  File "/usr/lib/python3.5/http/client.py", line 565, in _readall_chunked
   1.213 -    raise IncompleteRead(b''.join(value))
   1.214 -http.client.IncompleteRead: IncompleteRead(9713032 bytes read)
   1.215 -Traceback (most recent call last):
   1.216 -  File "<stdin>", line 1, in <module>
   1.217 -IndexError: list index out of range
   1.218 ->>>
   1.219 ---END--
   1.220 +    >   {
   1.221 +    >     "resources": [],
   1.222 +    >     "owner": "http://gillian:8080/firethorn/community-member/57698559",
   1.223 +    >     "text": null,
   1.224 +    >     "callback": "http://gillian:8081/firethorn/callback/57698652",
   1.225 +    >     "modified": "2019-05-08T19:34:59.347",
   1.226 +    >     "history": [],
   1.227 +    >     "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
   1.228 +    >     "delays": {
   1.229 +    >       "first": null,
   1.230 +    >       "last": null,
   1.231 +    >       "every": null
   1.232 +    >     },
   1.233 +    >     "url": "http://gillian:8080/firethorn/blue/query/57698652",
   1.234 +    >     "ident": "57698652",
   1.235 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
   1.236 +    >     "columns": [],
   1.237 +    >     "fields": [],
   1.238 +    >     "tables": [],
   1.239 +    >     "mode": "AUTO",
   1.240 +    >     "created": "2019-05-08T19:34:59.210",
   1.241 +    >     "name": "XX_5U5BMFLXCC4E4AAAAFVJR4I3RI",
   1.242 +    >     "results": {
   1.243 +    >       "count": 0,
   1.244 +    >       "state": "NONE",
   1.245 +    >       "formats": {
   1.246 +    >         "datatable": null,
   1.247 +    >         "votable": null
   1.248 +    >       },
   1.249 +    >       "table": null
   1.250 +    >     },
   1.251 +    >     "osql": null,
   1.252 +    >     "adql": null,
   1.253 +    >     "workspace": "http://gillian:8080/firethorn/adql/resource/57688319",
   1.254 +    >     "limits": {
   1.255 +    >       "cells": null,
   1.256 +    >       "rows": 1000000,
   1.257 +    >       "time": null
   1.258 +    >     },
   1.259 +    >     "resource": "http://gillian:8080/firethorn/adql/resource/57688319",
   1.260 +    >     "status": "FAILED",
   1.261 +    >     "syntax": {
   1.262 +    >       "status": "PARSE_ERROR",
   1.263 +    >       "friendly": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !",
   1.264 +    >       "message": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !"
   1.265 +    >     },
   1.266 +    >     "self": "http://gillian:8080/firethorn/blue/query/57698652"
   1.267 +    >   }
   1.268 +    >   >>> print(
   1.269 +    >   ...     query_obj.table()
   1.270 +    >   ...     )
   1.271 +    >   ERROR:root:unknown url type: 'None'
   1.272 +    >   Traceback (most recent call last):
   1.273 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.274 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.275 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.276 +    >       self.full_url = url
   1.277 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.278 +    >       self._parse()
   1.279 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.280 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.281 +    >   ValueError: unknown url type: 'None'
   1.282 +    >   ERROR:root:unknown url type: 'None'
   1.283 +    >   Traceback (most recent call last):
   1.284 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.285 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.286 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.287 +    >       self.full_url = url
   1.288 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.289 +    >       self._parse()
   1.290 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.291 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.292 +    >   ValueError: unknown url type: 'None'
   1.293 +    >   ERROR:root:unknown url type: 'None'
   1.294 +    >   Traceback (most recent call last):
   1.295 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.296 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.297 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.298 +    >       self.full_url = url
   1.299 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.300 +    >       self._parse()
   1.301 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.302 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.303 +    >   ValueError: unknown url type: 'None'
   1.304 +    >   null
   1.305 +    >   >>> print(
   1.306 +    >   ...     query_obj.table().count()
   1.307 +    >   ...     )
   1.308 +    >   ERROR:root:unknown url type: 'None'
   1.309 +    >   Traceback (most recent call last):
   1.310 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.311 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.312 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.313 +    >       self.full_url = url
   1.314 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.315 +    >       self._parse()
   1.316 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.317 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.318 +    >   ValueError: unknown url type: 'None'
   1.319 +    >   ERROR:root:unknown url type: 'None'
   1.320 +    >   Traceback (most recent call last):
   1.321 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.322 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.323 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.324 +    >       self.full_url = url
   1.325 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.326 +    >       self._parse()
   1.327 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.328 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.329 +    >   ValueError: unknown url type: 'None'
   1.330 +    >   ERROR:root:unknown url type: 'None'
   1.331 +    >   Traceback (most recent call last):
   1.332 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.333 +    >       req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.334 +    >     File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.335 +    >       self.full_url = url
   1.336 +    >     File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.337 +    >       self._parse()
   1.338 +    >     File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.339 +    >       raise ValueError("unknown url type: %r" % self.full_url)
   1.340 +    >   ValueError: unknown url type: 'None'
   1.341 +    >   None
   1.342 +    >   >>>
   1.343 +    >   >>>
   1.344 +    >   >>>
   1.345 +    >   >>>
   1.346 +    >   >>>
   1.347 +    >   >>> resource = firethorn.firethorn_engine.select_adql_resources()[28]
   1.348 +    >   ERROR:root:IncompleteRead(9713032 bytes read)
   1.349 +    >   Traceback (most recent call last):
   1.350 +    >     File "/usr/lib/python3.5/http/client.py", line 541, in _get_chunk_left
   1.351 +    >       chunk_left = self._read_next_chunk_size()
   1.352 +    >     File "/usr/lib/python3.5/http/client.py", line 508, in _read_next_chunk_size
   1.353 +    >       return int(line, 16)
   1.354 +    >   ValueError: invalid literal for int() with base 16: b''
   1.355 +    >   
   1.356 +    >   During handling of the above exception, another exception occurred:
   1.357 +    >   
   1.358 +    >   Traceback (most recent call last):
   1.359 +    >     File "/usr/lib/python3.5/http/client.py", line 558, in _readall_chunked
   1.360 +    >       chunk_left = self._get_chunk_left()
   1.361 +    >     File "/usr/lib/python3.5/http/client.py", line 543, in _get_chunk_left
   1.362 +    >       raise IncompleteRead(b'')
   1.363 +    >   http.client.IncompleteRead: IncompleteRead(0 bytes read)
   1.364 +    >   
   1.365 +    >   During handling of the above exception, another exception occurred:
   1.366 +    >   
   1.367 +    >   Traceback (most recent call last):
   1.368 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/core/firethorn_engine.py", line 238, in select_adql_resources
   1.369 +    >       adqlresources =  json.loads(response.read().decode('utf-8'))
   1.370 +    >     File "/usr/lib/python3.5/http/client.py", line 455, in read
   1.371 +    >       return self._readall_chunked()
   1.372 +    >     File "/usr/lib/python3.5/http/client.py", line 565, in _readall_chunked
   1.373 +    >       raise IncompleteRead(b''.join(value))
   1.374 +    >   http.client.IncompleteRead: IncompleteRead(9713032 bytes read)
   1.375 +    >   Traceback (most recent call last):
   1.376 +    >     File "<stdin>", line 1, in <module>
   1.377 +    >   IndexError: list index out of range
   1.378 +    >   >>>
   1.379  
   1.380  
   1.381  
   1.382 @@ -339,212 +333,208 @@
   1.383  
   1.384          tail -n 100 logs/firethorn-debug.log
   1.385  
   1.386 ---START--
   1.387 -2019-05-08 19:48:30,592 DEBUG [callback-interface-1] [AnonymousAuthenticator] Adding anonymous Authentication
   1.388 -2019-05-08 19:48:31,322 DEBUG [callback-interface-5] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.389 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   queryString [null]
   1.390 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   authType    [null]
   1.391 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.392 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug] Request headers
   1.393 -2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   [host][localhost:8081]
   1.394 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   authType    [null]
   1.395 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.396 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request headers
   1.397 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [host][localhost:8081]
   1.398 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.399 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [accept][*/*]
   1.400 -2019-05-08 19:48:32,659 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.401 -2019-05-08 19:48:34,236 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.402 -2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request parameters
   1.403 -2019-05-08 19:48:34,236 DEBUG [callback-interface-5] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.404 -2019-05-08 19:48:36,790 DEBUG [callback-interface-3] [HttpRequestDebug] ----
   1.405 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug]   [accept][*/*]
   1.406 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] Request parameters
   1.407 -2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] ----
   1.408 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug] Request properties
   1.409 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.410 -2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   queryString [null]
   1.411 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   authType    [null]
   1.412 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.413 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request headers
   1.414 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [host][localhost:8081]
   1.415 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.416 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [accept][*/*]
   1.417 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request parameters
   1.418 -2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] ----
   1.419 -2019-05-08 19:48:43,346 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.420 -2019-05-08 19:48:44,023 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.421 -2019-05-08 19:48:45,865 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.422 -2019-05-08 19:49:27,372 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.423 -2019-05-08 19:49:27,372 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!
   1.424 -2019-05-08 19:51:34,886 DEBUG [callback-interface-3] [AnonymousAuthenticator] Adding anonymous Authentication
   1.425 -2019-05-08 19:56:52,233 DEBUG [callback-interface-10] [HttpRequestDebug] Request properties
   1.426 -2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.427 -2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   queryString [null]
   1.428 -2019-05-08 19:56:58,423 DEBUG [callback-interface-10] [HttpRequestDebug]   authType    [null]
   1.429 -2019-05-08 19:57:02,194 DEBUG [callback-interface-10] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.430 -2019-05-08 19:57:02,936 DEBUG [callback-interface-10] [HttpRequestDebug] Request headers
   1.431 -2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [host][localhost:8081]
   1.432 -2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.433 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug]   [accept][*/*]
   1.434 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] Request parameters
   1.435 -2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] ----
   1.436 -2019-05-08 19:57:10,840 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Complete Status:
   1.437 -	Managed Threads: 3
   1.438 -	Active Threads: 3
   1.439 -	Active Tasks:
   1.440 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@481a442c (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0)
   1.441 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1)
   1.442 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@7998e972 (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2)
   1.443 -	Pending Tasks:
   1.444 -		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@3214a229
   1.445 -		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@2835bb63
   1.446 -Pool thread stack traces:
   1.447 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0,5,main]
   1.448 -		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.449 -		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.450 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.451 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.452 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1,5,main]
   1.453 -		java.util.zip.ZipFile.getEntry(ZipFile.java:314)
   1.454 -		java.util.jar.JarFile.getEntry(JarFile.java:240)
   1.455 -		java.util.jar.JarFile.getJarEntry(JarFile.java:223)
   1.456 -		org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:98)
   1.457 -		org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:256)
   1.458 -		org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281)
   1.459 -		org.apache.catalina.webresources.Cache.getResource(Cache.java:62)
   1.460 -		org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216)
   1.461 -		org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225)
   1.462 -		org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2241)
   1.463 -		org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:833)
   1.464 -		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1278)
   1.465 -		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1138)
   1.466 -		org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:250)
   1.467 -		org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
   1.468 -		org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
   1.469 -		org.postgresql.Driver.makeConnection(Driver.java:454)
   1.470 -		org.postgresql.Driver.connect(Driver.java:256)
   1.471 -		com.mchange.v2.c3p0.DriverManagerDataSource.getConnection(DriverManagerDataSource.java:134)
   1.472 -		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:182)
   1.473 -		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:171)
   1.474 -		com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool$1PooledConnectionResourcePoolManager.acquireResource(C3P0PooledConnectionPool.java:137)
   1.475 -		com.mchange.v2.resourcepool.BasicResourcePool.doAcquire(BasicResourcePool.java:1014)
   1.476 -		com.mchange.v2.resourcepool.BasicResourcePool.access$800(BasicResourcePool.java:32)
   1.477 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1810)
   1.478 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.479 -	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2,5,main]
   1.480 -		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.481 -		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.482 -		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.483 -		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.484 -
   1.485 -2019-05-08 19:59:26,545 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] Task com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (in deadlocked PoolThread) failed to complete in maximum time 60000ms. Trying interrupt().
   1.486 ---END--
   1.487 +    >   2019-05-08 19:48:30,592 DEBUG [callback-interface-1] [AnonymousAuthenticator] Adding anonymous Authentication
   1.488 +    >   2019-05-08 19:48:31,322 DEBUG [callback-interface-5] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.489 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   queryString [null]
   1.490 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   authType    [null]
   1.491 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.492 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug] Request headers
   1.493 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   [host][localhost:8081]
   1.494 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   authType    [null]
   1.495 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.496 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request headers
   1.497 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [host][localhost:8081]
   1.498 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.499 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [accept][*/*]
   1.500 +    >   2019-05-08 19:48:32,659 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.501 +    >   2019-05-08 19:48:34,236 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.502 +    >   2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request parameters
   1.503 +    >   2019-05-08 19:48:34,236 DEBUG [callback-interface-5] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.504 +    >   2019-05-08 19:48:36,790 DEBUG [callback-interface-3] [HttpRequestDebug] ----
   1.505 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug]   [accept][*/*]
   1.506 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] Request parameters
   1.507 +    >   2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] ----
   1.508 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug] Request properties
   1.509 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.510 +    >   2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   queryString [null]
   1.511 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   authType    [null]
   1.512 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.513 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request headers
   1.514 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [host][localhost:8081]
   1.515 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.516 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [accept][*/*]
   1.517 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request parameters
   1.518 +    >   2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] ----
   1.519 +    >   2019-05-08 19:48:43,346 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.520 +    >   2019-05-08 19:48:44,023 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.521 +    >   2019-05-08 19:48:45,865 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.522 +    >   2019-05-08 19:49:27,372 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.523 +    >   2019-05-08 19:49:27,372 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!
   1.524 +    >   2019-05-08 19:51:34,886 DEBUG [callback-interface-3] [AnonymousAuthenticator] Adding anonymous Authentication
   1.525 +    >   2019-05-08 19:56:52,233 DEBUG [callback-interface-10] [HttpRequestDebug] Request properties
   1.526 +    >   2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.527 +    >   2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   queryString [null]
   1.528 +    >   2019-05-08 19:56:58,423 DEBUG [callback-interface-10] [HttpRequestDebug]   authType    [null]
   1.529 +    >   2019-05-08 19:57:02,194 DEBUG [callback-interface-10] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.530 +    >   2019-05-08 19:57:02,936 DEBUG [callback-interface-10] [HttpRequestDebug] Request headers
   1.531 +    >   2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [host][localhost:8081]
   1.532 +    >   2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.533 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug]   [accept][*/*]
   1.534 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] Request parameters
   1.535 +    >   2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] ----
   1.536 +    >   2019-05-08 19:57:10,840 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Complete Status:
   1.537 +    >   	Managed Threads: 3
   1.538 +    >   	Active Threads: 3
   1.539 +    >   	Active Tasks:
   1.540 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@481a442c (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0)
   1.541 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1)
   1.542 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@7998e972 (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2)
   1.543 +    >   	Pending Tasks:
   1.544 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@3214a229
   1.545 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@2835bb63
   1.546 +    >   Pool thread stack traces:
   1.547 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0,5,main]
   1.548 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.549 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.550 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.551 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.552 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1,5,main]
   1.553 +    >   		java.util.zip.ZipFile.getEntry(ZipFile.java:314)
   1.554 +    >   		java.util.jar.JarFile.getEntry(JarFile.java:240)
   1.555 +    >   		java.util.jar.JarFile.getJarEntry(JarFile.java:223)
   1.556 +    >   		org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:98)
   1.557 +    >   		org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:256)
   1.558 +    >   		org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281)
   1.559 +    >   		org.apache.catalina.webresources.Cache.getResource(Cache.java:62)
   1.560 +    >   		org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216)
   1.561 +    >   		org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225)
   1.562 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2241)
   1.563 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:833)
   1.564 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1278)
   1.565 +    >   		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1138)
   1.566 +    >   		org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:250)
   1.567 +    >   		org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
   1.568 +    >   		org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
   1.569 +    >   		org.postgresql.Driver.makeConnection(Driver.java:454)
   1.570 +    >   		org.postgresql.Driver.connect(Driver.java:256)
   1.571 +    >   		com.mchange.v2.c3p0.DriverManagerDataSource.getConnection(DriverManagerDataSource.java:134)
   1.572 +    >   		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:182)
   1.573 +    >   		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:171)
   1.574 +    >   		com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool$1PooledConnectionResourcePoolManager.acquireResource(C3P0PooledConnectionPool.java:137)
   1.575 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.doAcquire(BasicResourcePool.java:1014)
   1.576 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$800(BasicResourcePool.java:32)
   1.577 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1810)
   1.578 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.579 +    >   	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2,5,main]
   1.580 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.581 +    >   		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.582 +    >   		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.583 +    >   		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.584 +    >   
   1.585 +    >   2019-05-08 19:59:26,545 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] Task com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (in deadlocked PoolThread) failed to complete in maximum time 60000ms. Trying interrupt().
   1.586  
   1.587      tail -n 100 logs/firethorn-trace.log
   1.588  
   1.589 ---START--
   1.590 -2019-05-08 20:17:33,686 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.591 -2019-05-08 20:17:36,580 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754910]
   1.592 -2019-05-08 20:17:36,580 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.593 -2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.594 -2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.595 -2019-05-08 20:17:40,212 TRACE [main-interface-2] [IdentityEntity] services()
   1.596 -2019-05-08 20:17:40,213 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.597 -2019-05-08 20:17:42,353 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.598 -2019-05-08 20:17:43,793 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754909]
   1.599 -2019-05-08 20:17:44,500 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.600 -2019-05-08 20:17:45,886 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.601 -2019-05-08 20:17:47,983 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.602 -2019-05-08 20:17:53,135 TRACE [main-interface-2] [IdentityEntity] services()
   1.603 -2019-05-08 20:17:54,679 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.604 -2019-05-08 20:17:57,601 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.605 -2019-05-08 20:18:01,181 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754908]
   1.606 -2019-05-08 20:18:01,897 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.607 -2019-05-08 20:18:03,326 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.608 -2019-05-08 20:18:04,806 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.609 -2019-05-08 20:18:09,791 TRACE [main-interface-2] [IdentityEntity] services()
   1.610 -2019-05-08 20:18:12,605 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.611 -2019-05-08 20:18:24,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.612 -2019-05-08 20:18:27,672 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754907]
   1.613 -2019-05-08 20:18:27,672 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.614 -2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.615 -2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.616 -2019-05-08 20:18:35,729 TRACE [main-interface-2] [IdentityEntity] services()
   1.617 -2019-05-08 20:18:38,654 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.618 -2019-05-08 20:18:40,785 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.619 -2019-05-08 20:18:44,659 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754906]
   1.620 -2019-05-08 20:18:44,659 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.621 -2019-05-08 20:18:47,137 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.622 -2019-05-08 20:18:49,387 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.623 -2019-05-08 20:18:53,563 TRACE [main-interface-2] [IdentityEntity] services()
   1.624 -2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.625 -2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.626 -2019-05-08 20:18:58,588 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754905]
   1.627 -2019-05-08 20:18:59,319 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.628 -2019-05-08 20:19:00,056 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.629 -2019-05-08 20:19:01,598 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.630 -2019-05-08 20:19:02,341 TRACE [main-interface-2] [IdentityEntity] services()
   1.631 -2019-05-08 20:19:05,328 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.632 -2019-05-08 20:19:06,127 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.633 -2019-05-08 20:19:10,942 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754904]
   1.634 -2019-05-08 20:19:14,163 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.635 -2019-05-08 20:19:17,369 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.636 -2019-05-08 20:19:26,151 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.637 -2019-05-08 20:19:30,374 TRACE [main-interface-2] [IdentityEntity] services()
   1.638 -2019-05-08 20:19:32,527 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.639 -2019-05-08 20:19:34,588 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.640 -2019-05-08 20:19:37,357 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754903]
   1.641 -2019-05-08 20:19:40,885 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.642 -2019-05-08 20:19:44,329 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.643 -2019-05-08 20:19:53,642 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.644 -2019-05-08 20:19:57,191 TRACE [main-interface-2] [IdentityEntity] services()
   1.645 -2019-05-08 20:20:00,019 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.646 -2019-05-08 20:20:10,334 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.647 -2019-05-08 20:20:20,901 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754902]
   1.648 -2019-05-08 20:20:21,729 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.649 -2019-05-08 20:20:24,094 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.650 -2019-05-08 20:20:24,873 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.651 -2019-05-08 20:20:28,789 TRACE [main-interface-2] [IdentityEntity] services()
   1.652 -2019-05-08 20:20:29,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.653 -2019-05-08 20:20:39,949 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.654 -2019-05-08 20:20:47,558 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750801]
   1.655 -2019-05-08 20:20:47,558 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.656 -2019-05-08 20:20:48,242 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.657 -2019-05-08 20:20:50,360 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.658 -2019-05-08 20:20:53,869 TRACE [main-interface-2] [IdentityEntity] services()
   1.659 -2019-05-08 20:20:56,645 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.660 -2019-05-08 20:20:59,454 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.661 -2019-05-08 20:21:04,386 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750800]
   1.662 -2019-05-08 20:21:04,386 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.663 -2019-05-08 20:21:06,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.664 -2019-05-08 20:21:09,381 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.665 -2019-05-08 20:21:12,257 TRACE [main-interface-2] [IdentityEntity] services()
   1.666 -2019-05-08 20:21:13,696 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.667 -2019-05-08 20:21:16,517 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.668 -2019-05-08 20:21:21,786 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750799]
   1.669 -2019-05-08 20:21:21,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.670 -2019-05-08 20:21:23,973 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.671 -2019-05-08 20:21:26,908 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.672 -2019-05-08 20:21:31,215 TRACE [main-interface-2] [IdentityEntity] services()
   1.673 -2019-05-08 20:21:34,123 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.674 -2019-05-08 20:21:37,228 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.675 -2019-05-08 20:21:43,020 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750798]
   1.676 -2019-05-08 20:21:45,862 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.677 -2019-05-08 20:21:47,251 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.678 -2019-05-08 20:21:50,799 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.679 -2019-05-08 20:21:54,968 TRACE [main-interface-2] [IdentityEntity] services()
   1.680 -2019-05-08 20:21:59,238 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.681 -2019-05-08 20:22:02,876 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.682 -2019-05-08 20:22:19,458 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750797]
   1.683 -2019-05-08 20:22:20,978 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.684 -2019-05-08 20:22:25,532 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.685 -2019-05-08 20:22:30,615 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.686 -2019-05-08 20:24:14,090 TRACE [main-interface-2] [AnonymousAuthenticator] afterCompletion()
   1.687 -2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] afterCompletion()
   1.688 -2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] Operation [57698710][http://gillian:8080/firethorn/adql/resource/select]
   1.689 -2019-05-08 20:24:28,569 DEBUG [main-interface-2] [OperationInterceptor] Exception [org.springframework.web.util.NestedServletException][Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space]
   1.690 ---END--
   1.691 +    >   2019-05-08 20:17:33,686 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.692 +    >   2019-05-08 20:17:36,580 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754910]
   1.693 +    >   2019-05-08 20:17:36,580 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.694 +    >   2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.695 +    >   2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.696 +    >   2019-05-08 20:17:40,212 TRACE [main-interface-2] [IdentityEntity] services()
   1.697 +    >   2019-05-08 20:17:40,213 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.698 +    >   2019-05-08 20:17:42,353 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.699 +    >   2019-05-08 20:17:43,793 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754909]
   1.700 +    >   2019-05-08 20:17:44,500 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.701 +    >   2019-05-08 20:17:45,886 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.702 +    >   2019-05-08 20:17:47,983 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.703 +    >   2019-05-08 20:17:53,135 TRACE [main-interface-2] [IdentityEntity] services()
   1.704 +    >   2019-05-08 20:17:54,679 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.705 +    >   2019-05-08 20:17:57,601 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.706 +    >   2019-05-08 20:18:01,181 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754908]
   1.707 +    >   2019-05-08 20:18:01,897 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.708 +    >   2019-05-08 20:18:03,326 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.709 +    >   2019-05-08 20:18:04,806 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.710 +    >   2019-05-08 20:18:09,791 TRACE [main-interface-2] [IdentityEntity] services()
   1.711 +    >   2019-05-08 20:18:12,605 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.712 +    >   2019-05-08 20:18:24,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.713 +    >   2019-05-08 20:18:27,672 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754907]
   1.714 +    >   2019-05-08 20:18:27,672 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.715 +    >   2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.716 +    >   2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.717 +    >   2019-05-08 20:18:35,729 TRACE [main-interface-2] [IdentityEntity] services()
   1.718 +    >   2019-05-08 20:18:38,654 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.719 +    >   2019-05-08 20:18:40,785 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.720 +    >   2019-05-08 20:18:44,659 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754906]
   1.721 +    >   2019-05-08 20:18:44,659 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.722 +    >   2019-05-08 20:18:47,137 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.723 +    >   2019-05-08 20:18:49,387 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.724 +    >   2019-05-08 20:18:53,563 TRACE [main-interface-2] [IdentityEntity] services()
   1.725 +    >   2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.726 +    >   2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.727 +    >   2019-05-08 20:18:58,588 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754905]
   1.728 +    >   2019-05-08 20:18:59,319 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.729 +    >   2019-05-08 20:19:00,056 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.730 +    >   2019-05-08 20:19:01,598 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.731 +    >   2019-05-08 20:19:02,341 TRACE [main-interface-2] [IdentityEntity] services()
   1.732 +    >   2019-05-08 20:19:05,328 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.733 +    >   2019-05-08 20:19:06,127 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.734 +    >   2019-05-08 20:19:10,942 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754904]
   1.735 +    >   2019-05-08 20:19:14,163 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.736 +    >   2019-05-08 20:19:17,369 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.737 +    >   2019-05-08 20:19:26,151 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.738 +    >   2019-05-08 20:19:30,374 TRACE [main-interface-2] [IdentityEntity] services()
   1.739 +    >   2019-05-08 20:19:32,527 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.740 +    >   2019-05-08 20:19:34,588 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.741 +    >   2019-05-08 20:19:37,357 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754903]
   1.742 +    >   2019-05-08 20:19:40,885 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.743 +    >   2019-05-08 20:19:44,329 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.744 +    >   2019-05-08 20:19:53,642 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.745 +    >   2019-05-08 20:19:57,191 TRACE [main-interface-2] [IdentityEntity] services()
   1.746 +    >   2019-05-08 20:20:00,019 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.747 +    >   2019-05-08 20:20:10,334 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.748 +    >   2019-05-08 20:20:20,901 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754902]
   1.749 +    >   2019-05-08 20:20:21,729 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.750 +    >   2019-05-08 20:20:24,094 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.751 +    >   2019-05-08 20:20:24,873 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.752 +    >   2019-05-08 20:20:28,789 TRACE [main-interface-2] [IdentityEntity] services()
   1.753 +    >   2019-05-08 20:20:29,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.754 +    >   2019-05-08 20:20:39,949 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.755 +    >   2019-05-08 20:20:47,558 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750801]
   1.756 +    >   2019-05-08 20:20:47,558 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.757 +    >   2019-05-08 20:20:48,242 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.758 +    >   2019-05-08 20:20:50,360 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.759 +    >   2019-05-08 20:20:53,869 TRACE [main-interface-2] [IdentityEntity] services()
   1.760 +    >   2019-05-08 20:20:56,645 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.761 +    >   2019-05-08 20:20:59,454 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.762 +    >   2019-05-08 20:21:04,386 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750800]
   1.763 +    >   2019-05-08 20:21:04,386 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.764 +    >   2019-05-08 20:21:06,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.765 +    >   2019-05-08 20:21:09,381 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.766 +    >   2019-05-08 20:21:12,257 TRACE [main-interface-2] [IdentityEntity] services()
   1.767 +    >   2019-05-08 20:21:13,696 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.768 +    >   2019-05-08 20:21:16,517 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.769 +    >   2019-05-08 20:21:21,786 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750799]
   1.770 +    >   2019-05-08 20:21:21,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.771 +    >   2019-05-08 20:21:23,973 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.772 +    >   2019-05-08 20:21:26,908 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.773 +    >   2019-05-08 20:21:31,215 TRACE [main-interface-2] [IdentityEntity] services()
   1.774 +    >   2019-05-08 20:21:34,123 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.775 +    >   2019-05-08 20:21:37,228 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.776 +    >   2019-05-08 20:21:43,020 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750798]
   1.777 +    >   2019-05-08 20:21:45,862 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.778 +    >   2019-05-08 20:21:47,251 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.779 +    >   2019-05-08 20:21:50,799 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.780 +    >   2019-05-08 20:21:54,968 TRACE [main-interface-2] [IdentityEntity] services()
   1.781 +    >   2019-05-08 20:21:59,238 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.782 +    >   2019-05-08 20:22:02,876 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.783 +    >   2019-05-08 20:22:19,458 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750797]
   1.784 +    >   2019-05-08 20:22:20,978 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.785 +    >   2019-05-08 20:22:25,532 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.786 +    >   2019-05-08 20:22:30,615 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.787 +    >   2019-05-08 20:24:14,090 TRACE [main-interface-2] [AnonymousAuthenticator] afterCompletion()
   1.788 +    >   2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] afterCompletion()
   1.789 +    >   2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] Operation [57698710][http://gillian:8080/firethorn/adql/resource/select]
   1.790 +    >   2019-05-08 20:24:28,569 DEBUG [main-interface-2] [OperationInterceptor] Exception [org.springframework.web.util.NestedServletException][Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space]
   1.791  
   1.792  # -----------------------------------------------------
   1.793  
   1.794 @@ -561,23 +551,21 @@
   1.795  
   1.796      df -h
   1.797  
   1.798 ---START--
   1.799 -Filesystem      Size  Used Avail Use% Mounted on
   1.800 -/dev/sda2        92G   58G   30G  66% /
   1.801 -udev             10M     0   10M   0% /dev
   1.802 -tmpfs            26G  1.2G   25G   5% /run
   1.803 -tmpfs            64G     0   64G   0% /dev/shm
   1.804 -tmpfs           5.0M     0  5.0M   0% /run/lock
   1.805 -tmpfs            64G     0   64G   0% /sys/fs/cgroup
   1.806 -/dev/sda1       511M  132K  511M   1% /boot/efi
   1.807 -/dev/sda4       9.1G   22M  8.6G   1% /tmp
   1.808 -/dev/sda5        65G  5.4G   56G   9% /var
   1.809 -/dev/sda6        53G   53M   50G   1% /home
   1.810 -/dev/sdc1       3.6T   68M  3.4T   1% /data2
   1.811 -/dev/sdb1       3.6T   74M  3.4T   1% /data1
   1.812 -tmpfs            13G     0   13G   0% /run/user/1005
   1.813 -tmpfs            13G     0   13G   0% /run/user/0
   1.814 ---END--
   1.815 +    >   Filesystem      Size  Used Avail Use% Mounted on
   1.816 +    >   /dev/sda2        92G   58G   30G  66% /
   1.817 +    >   udev             10M     0   10M   0% /dev
   1.818 +    >   tmpfs            26G  1.2G   25G   5% /run
   1.819 +    >   tmpfs            64G     0   64G   0% /dev/shm
   1.820 +    >   tmpfs           5.0M     0  5.0M   0% /run/lock
   1.821 +    >   tmpfs            64G     0   64G   0% /sys/fs/cgroup
   1.822 +    >   /dev/sda1       511M  132K  511M   1% /boot/efi
   1.823 +    >   /dev/sda4       9.1G   22M  8.6G   1% /tmp
   1.824 +    >   /dev/sda5        65G  5.4G   56G   9% /var
   1.825 +    >   /dev/sda6        53G   53M   50G   1% /home
   1.826 +    >   /dev/sdc1       3.6T   68M  3.4T   1% /data2
   1.827 +    >   /dev/sdb1       3.6T   74M  3.4T   1% /data1
   1.828 +    >   tmpfs            13G     0   13G   0% /run/user/1005
   1.829 +    >   tmpfs            13G     0   13G   0% /run/user/0
   1.830  
   1.831  # -----------------------------------------------------
   1.832  # Memory on the host is not the issue (29G cached).
   1.833 @@ -585,12 +573,10 @@
   1.834  
   1.835      free -h
   1.836  
   1.837 ---START--
   1.838 -             total       used       free     shared    buffers     cached
   1.839 -Mem:          126G        39G        86G       1.1G       297M        29G
   1.840 --/+ buffers/cache:       9.3G       116G
   1.841 -Swap:         1.9G         0B       1.9G
   1.842 ---END--
   1.843 +    >                total       used       free     shared    buffers     cached
   1.844 +    >   Mem:          126G        39G        86G       1.1G       297M        29G
   1.845 +    >   -/+ buffers/cache:       9.3G       116G
   1.846 +    >   Swap:         1.9G         0B       1.9G
   1.847  
   1.848  
   1.849  # -----------------------------------------------------
   1.850 @@ -599,17 +585,15 @@
   1.851  
   1.852      df -h
   1.853  
   1.854 ---START--
   1.855 -Filesystem      Size  Used Avail Use% Mounted on
   1.856 -devtmpfs        2.0G     0  2.0G   0% /dev
   1.857 -tmpfs           2.0G     0  2.0G   0% /dev/shm
   1.858 -tmpfs           2.0G  940K  2.0G   1% /run
   1.859 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.860 -/dev/vda3        31G   23G  7.7G  75% /
   1.861 -tmpfs           2.0G   25M  2.0G   2% /tmp
   1.862 -/dev/vda1       240M   89M  135M  40% /boot
   1.863 -tmpfs           395M     0  395M   0% /run/user/1001
   1.864 ---END--
   1.865 +    >   Filesystem      Size  Used Avail Use% Mounted on
   1.866 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
   1.867 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
   1.868 +    >   tmpfs           2.0G  940K  2.0G   1% /run
   1.869 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.870 +    >   /dev/vda3        31G   23G  7.7G  75% /
   1.871 +    >   tmpfs           2.0G   25M  2.0G   2% /tmp
   1.872 +    >   /dev/vda1       240M   89M  135M  40% /boot
   1.873 +    >   tmpfs           395M     0  395M   0% /run/user/1001
   1.874  
   1.875  
   1.876  # -----------------------------------------------------
   1.877 @@ -618,11 +602,9 @@
   1.878  
   1.879      free -h
   1.880  
   1.881 ---START--
   1.882 -              total        used        free      shared  buff/cache   available
   1.883 -Mem:           3.9G        2.1G        125M        158M        1.7G        1.4G
   1.884 -Swap:          1.0G        174M        849M
   1.885 ---END--
   1.886 +    >                 total        used        free      shared  buff/cache   available
   1.887 +    >   Mem:           3.9G        2.1G        125M        158M        1.7G        1.4G
   1.888 +    >   Swap:          1.0G        174M        849M
   1.889  
   1.890  # -----------------------------------------------------
   1.891  # Disc space in the container is not the issue (75% used).
   1.892 @@ -630,19 +612,17 @@
   1.893  
   1.894      df -h
   1.895  
   1.896 ---START--
   1.897 -Filesystem      Size  Used Avail Use% Mounted on
   1.898 -/dev/vda3        31G   23G  7.7G  75% /
   1.899 -tmpfs            64M     0   64M   0% /dev
   1.900 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.901 -tmpfs           2.0G     0  2.0G   0% /run
   1.902 -tmpfs           2.0G   32K  2.0G   1% /tmp
   1.903 -/dev/vda3        31G   23G  7.7G  75% /etc/firethorn.properties
   1.904 -shm              64M     0   64M   0% /dev/shm
   1.905 -tmpfs           2.0G     0  2.0G   0% /proc/acpi
   1.906 -tmpfs           2.0G     0  2.0G   0% /proc/scsi
   1.907 -tmpfs           2.0G     0  2.0G   0% /sys/firmware
   1.908 ---END--
   1.909 +    >   Filesystem      Size  Used Avail Use% Mounted on
   1.910 +    >   /dev/vda3        31G   23G  7.7G  75% /
   1.911 +    >   tmpfs            64M     0   64M   0% /dev
   1.912 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.913 +    >   tmpfs           2.0G     0  2.0G   0% /run
   1.914 +    >   tmpfs           2.0G   32K  2.0G   1% /tmp
   1.915 +    >   /dev/vda3        31G   23G  7.7G  75% /etc/firethorn.properties
   1.916 +    >   shm              64M     0   64M   0% /dev/shm
   1.917 +    >   tmpfs           2.0G     0  2.0G   0% /proc/acpi
   1.918 +    >   tmpfs           2.0G     0  2.0G   0% /proc/scsi
   1.919 +    >   tmpfs           2.0G     0  2.0G   0% /sys/firmware
   1.920  
   1.921  
   1.922  # -----------------------------------------------------
   1.923 @@ -656,12 +636,10 @@
   1.924              --username "${metauser:?}" \
   1.925              --dbname   "${metadata:?}"
   1.926  
   1.927 ---START--
   1.928 -psql (10.6 (Debian 10.6-1.pgdg90+1))
   1.929 -Type "help" for help.
   1.930 -
   1.931 -postgres=#
   1.932 ---END--
   1.933 +    >   psql (10.6 (Debian 10.6-1.pgdg90+1))
   1.934 +    >   Type "help" for help.
   1.935 +    >   
   1.936 +    >   postgres=#
   1.937  
   1.938  
   1.939  # -----------------------------------------------------
   1.940 @@ -670,39 +648,37 @@
   1.941  
   1.942      \dt
   1.943  
   1.944 ---START--
   1.945 -                                 List of relations
   1.946 - Schema |                   Name                    | Type  |        Owner
   1.947 ---------+-------------------------------------------+-------+----------------------
   1.948 - public | ft020116adqlcolumnentity                  | table | liengooXien1mooP0tae
   1.949 - public | ft020116adqlresourceentity                | table | liengooXien1mooP0tae
   1.950 - public | ft020116adqlschemaentity                  | table | liengooXien1mooP0tae
   1.951 - public | ft020116adqltableentity                   | table | liengooXien1mooP0tae
   1.952 - public | ft020116authmethodentity                  | table | liengooXien1mooP0tae
   1.953 - public | ft020116bluequeryentity                   | table | liengooXien1mooP0tae
   1.954 - public | ft020116bluequeryentityjointobaseresource | table | liengooXien1mooP0tae
   1.955 - public | ft020116bluetaskentity                    | table | liengooXien1mooP0tae
   1.956 - public | ft020116bluetasklogentity                 | table | liengooXien1mooP0tae
   1.957 - public | ft020116bluetaskparam                     | table | liengooXien1mooP0tae
   1.958 - public | ft020116communityentity                   | table | liengooXien1mooP0tae
   1.959 - public | ft020116configproperty                    | table | liengooXien1mooP0tae
   1.960 - public | ft020116identityentity                    | table | liengooXien1mooP0tae
   1.961 - public | ft020116ivoacolumnentity                  | table | liengooXien1mooP0tae
   1.962 - public | ft020116ivoaresourceentity                | table | liengooXien1mooP0tae
   1.963 - public | ft020116ivoaschemaentity                  | table | liengooXien1mooP0tae
   1.964 - public | ft020116ivoatableentity                   | table | liengooXien1mooP0tae
   1.965 - public | ft020116jdbccolumnentity                  | table | liengooXien1mooP0tae
   1.966 - public | ft020116jdbcresourceentity                | table | liengooXien1mooP0tae
   1.967 - public | ft020116jdbcschemaentity                  | table | liengooXien1mooP0tae
   1.968 - public | ft020116jdbctableentity                   | table | liengooXien1mooP0tae
   1.969 - public | ft020116ogsadqpresourceentity             | table | liengooXien1mooP0tae
   1.970 - public | ft020116ogsaexecresourceentity            | table | liengooXien1mooP0tae
   1.971 - public | ft020116ogsaivoaresourceentity            | table | liengooXien1mooP0tae
   1.972 - public | ft020116ogsajdbcresourceentity            | table | liengooXien1mooP0tae
   1.973 - public | ft020116ogsaserviceentity                 | table | liengooXien1mooP0tae
   1.974 - public | ft020116operationentity                   | table | liengooXien1mooP0tae
   1.975 -(27 rows)
   1.976 ---END--
   1.977 +    >                                    List of relations
   1.978 +    >    Schema |                   Name                    | Type  |        Owner
   1.979 +    >   --------+-------------------------------------------+-------+----------------------
   1.980 +    >    public | ft020116adqlcolumnentity                  | table | liengooXien1mooP0tae
   1.981 +    >    public | ft020116adqlresourceentity                | table | liengooXien1mooP0tae
   1.982 +    >    public | ft020116adqlschemaentity                  | table | liengooXien1mooP0tae
   1.983 +    >    public | ft020116adqltableentity                   | table | liengooXien1mooP0tae
   1.984 +    >    public | ft020116authmethodentity                  | table | liengooXien1mooP0tae
   1.985 +    >    public | ft020116bluequeryentity                   | table | liengooXien1mooP0tae
   1.986 +    >    public | ft020116bluequeryentityjointobaseresource | table | liengooXien1mooP0tae
   1.987 +    >    public | ft020116bluetaskentity                    | table | liengooXien1mooP0tae
   1.988 +    >    public | ft020116bluetasklogentity                 | table | liengooXien1mooP0tae
   1.989 +    >    public | ft020116bluetaskparam                     | table | liengooXien1mooP0tae
   1.990 +    >    public | ft020116communityentity                   | table | liengooXien1mooP0tae
   1.991 +    >    public | ft020116configproperty                    | table | liengooXien1mooP0tae
   1.992 +    >    public | ft020116identityentity                    | table | liengooXien1mooP0tae
   1.993 +    >    public | ft020116ivoacolumnentity                  | table | liengooXien1mooP0tae
   1.994 +    >    public | ft020116ivoaresourceentity                | table | liengooXien1mooP0tae
   1.995 +    >    public | ft020116ivoaschemaentity                  | table | liengooXien1mooP0tae
   1.996 +    >    public | ft020116ivoatableentity                   | table | liengooXien1mooP0tae
   1.997 +    >    public | ft020116jdbccolumnentity                  | table | liengooXien1mooP0tae
   1.998 +    >    public | ft020116jdbcresourceentity                | table | liengooXien1mooP0tae
   1.999 +    >    public | ft020116jdbcschemaentity                  | table | liengooXien1mooP0tae
  1.1000 +    >    public | ft020116jdbctableentity                   | table | liengooXien1mooP0tae
  1.1001 +    >    public | ft020116ogsadqpresourceentity             | table | liengooXien1mooP0tae
  1.1002 +    >    public | ft020116ogsaexecresourceentity            | table | liengooXien1mooP0tae
  1.1003 +    >    public | ft020116ogsaivoaresourceentity            | table | liengooXien1mooP0tae
  1.1004 +    >    public | ft020116ogsajdbcresourceentity            | table | liengooXien1mooP0tae
  1.1005 +    >    public | ft020116ogsaserviceentity                 | table | liengooXien1mooP0tae
  1.1006 +    >    public | ft020116operationentity                   | table | liengooXien1mooP0tae
  1.1007 +    >   (27 rows)
  1.1008  
  1.1009  # -----------------------------------------------------
  1.1010  # Check the how big the tables are.
  1.1011 @@ -710,235 +686,183 @@
  1.1012  
  1.1013      SELECT COUNT(*) FROM ft020116adqlcolumnentity ;
  1.1014  
  1.1015 ---START--
  1.1016 -  count
  1.1017 ----------
  1.1018 - 4431653
  1.1019 ---END--
  1.1020 +    >     count
  1.1021 +    >   ---------
  1.1022 +    >    4431653
  1.1023  
  1.1024      SELECT COUNT(*) FROM ft020116adqlschemaentity ;
  1.1025  
  1.1026 ---START--
  1.1027 - count
  1.1028 ---------
  1.1029 - 124803
  1.1030 -(1 row)
  1.1031 ---END--
  1.1032 +    >    count
  1.1033 +    >   --------
  1.1034 +    >    124803
  1.1035 +    >   (1 row)
  1.1036  
  1.1037      SELECT COUNT(*) FROM ft020116adqltableentity ;
  1.1038  
  1.1039 ---START--
  1.1040 - count
  1.1041 ---------
  1.1042 - 166863
  1.1043 -(1 row)
  1.1044 ---END--
  1.1045 +    >    count
  1.1046 +    >   --------
  1.1047 +    >    166863
  1.1048 +    >   (1 row)
  1.1049  
  1.1050      SELECT COUNT(*) FROM ft020116authmethodentity ;
  1.1051  
  1.1052 ---START--
  1.1053 -  count
  1.1054 -----------
  1.1055 - 16020562
  1.1056 ---END--
  1.1057 +    >     count
  1.1058 +    >   ----------
  1.1059 +    >    16020562
  1.1060  
  1.1061      SELECT COUNT(*) FROM ft020116bluequeryentity ;
  1.1062  
  1.1063 ---START--
  1.1064 - count
  1.1065 ---------
  1.1066 - 171174
  1.1067 -(1 row)
  1.1068 ---END--
  1.1069 +    >    count
  1.1070 +    >   --------
  1.1071 +    >    171174
  1.1072 +    >   (1 row)
  1.1073  
  1.1074      SELECT COUNT(*) FROM ft020116bluequeryentityjointobaseresource ;
  1.1075  
  1.1076 ---START--
  1.1077 - count
  1.1078 ---------
  1.1079 - 165455
  1.1080 -(1 row)
  1.1081 ---END--
  1.1082 +    >    count
  1.1083 +    >   --------
  1.1084 +    >    165455
  1.1085 +    >   (1 row)
  1.1086  
  1.1087      SELECT COUNT(*) FROM ft020116bluetaskentity ;
  1.1088  
  1.1089 ---START--
  1.1090 - count
  1.1091 ---------
  1.1092 - 171174
  1.1093 -(1 row)
  1.1094 ---END--
  1.1095 +    >    count
  1.1096 +    >   --------
  1.1097 +    >    171174
  1.1098 +    >   (1 row)
  1.1099  
  1.1100      SELECT COUNT(*) FROM ft020116bluetasklogentity ;
  1.1101  
  1.1102 ---START--
  1.1103 - count
  1.1104 ---------
  1.1105 - 552679
  1.1106 -(1 row)
  1.1107 ---END--
  1.1108 +    >    count
  1.1109 +    >   --------
  1.1110 +    >    552679
  1.1111 +    >   (1 row)
  1.1112  
  1.1113      SELECT COUNT(*) FROM ft020116bluetaskparam ;
  1.1114  
  1.1115 ---START--
  1.1116 - count
  1.1117 --------
  1.1118 -  2832
  1.1119 -(1 row)
  1.1120 ---END--
  1.1121 +    >    count
  1.1122 +    >   -------
  1.1123 +    >     2832
  1.1124 +    >   (1 row)
  1.1125  
  1.1126      SELECT COUNT(*) FROM ft020116communityentity ;
  1.1127  
  1.1128 ---START--
  1.1129 - count
  1.1130 --------
  1.1131 -     2
  1.1132 -(1 row)
  1.1133 ---END--
  1.1134 +    >    count
  1.1135 +    >   -------
  1.1136 +    >        2
  1.1137 +    >   (1 row)
  1.1138  
  1.1139      SELECT COUNT(*) FROM ft020116configproperty ;
  1.1140  
  1.1141 ---START--
  1.1142 - count
  1.1143 --------
  1.1144 -     0
  1.1145 -(1 row)
  1.1146 ---END--
  1.1147 +    >    count
  1.1148 +    >   -------
  1.1149 +    >        0
  1.1150 +    >   (1 row)
  1.1151  
  1.1152      SELECT COUNT(*) FROM ft020116identityentity ;
  1.1153  
  1.1154 ---START--
  1.1155 -  count
  1.1156 -----------
  1.1157 - 15313504
  1.1158 -(1 row)
  1.1159 ---END--
  1.1160 +    >     count
  1.1161 +    >   ----------
  1.1162 +    >    15313504
  1.1163 +    >   (1 row)
  1.1164  
  1.1165      SELECT COUNT(*) FROM ft020116ivoacolumnentity ;
  1.1166  
  1.1167 ---START--
  1.1168 - count
  1.1169 --------
  1.1170 -     0
  1.1171 -(1 row)
  1.1172 ---END--
  1.1173 +    >    count
  1.1174 +    >   -------
  1.1175 +    >        0
  1.1176 +    >   (1 row)
  1.1177  
  1.1178      SELECT COUNT(*) FROM ft020116ivoaresourceentity ;
  1.1179  
  1.1180 ---START--
  1.1181 - count
  1.1182 --------
  1.1183 -     0
  1.1184 -(1 row)
  1.1185 ---END--
  1.1186 +    >    count
  1.1187 +    >   -------
  1.1188 +    >        0
  1.1189 +    >   (1 row)
  1.1190  
  1.1191      SELECT COUNT(*) FROM ft020116ivoaschemaentity ;
  1.1192  
  1.1193 ---START--
  1.1194 - count
  1.1195 --------
  1.1196 -     0
  1.1197 -(1 row)
  1.1198 ---END--
  1.1199 +    >    count
  1.1200 +    >   -------
  1.1201 +    >        0
  1.1202 +    >   (1 row)
  1.1203  
  1.1204      SELECT COUNT(*) FROM ft020116ivoatableentity ;
  1.1205  
  1.1206 ---START--
  1.1207 - count
  1.1208 --------
  1.1209 -     0
  1.1210 -(1 row)
  1.1211 ---END--
  1.1212 +    >    count
  1.1213 +    >   -------
  1.1214 +    >        0
  1.1215 +    >   (1 row)
  1.1216  
  1.1217      SELECT COUNT(*) FROM ft020116jdbccolumnentity ;
  1.1218  
  1.1219 ---START--
  1.1220 -  count
  1.1221 ----------
  1.1222 - 4431979
  1.1223 -(1 row)
  1.1224 ---END--
  1.1225 +    >     count
  1.1226 +    >   ---------
  1.1227 +    >    4431979
  1.1228 +    >   (1 row)
  1.1229  
  1.1230      SELECT COUNT(*) FROM ft020116jdbcresourceentity ;
  1.1231  
  1.1232 ---START--
  1.1233 - count
  1.1234 --------
  1.1235 -    15
  1.1236 -(1 row)
  1.1237 ---END--
  1.1238 +    >    count
  1.1239 +    >   -------
  1.1240 +    >       15
  1.1241 +    >   (1 row)
  1.1242  
  1.1243      SELECT COUNT(*) FROM ft020116jdbcschemaentity ;
  1.1244  
  1.1245 ---START--
  1.1246 - count
  1.1247 --------
  1.1248 -  1206
  1.1249 -(1 row)
  1.1250 ---END--
  1.1251 +    >    count
  1.1252 +    >   -------
  1.1253 +    >     1206
  1.1254 +    >   (1 row)
  1.1255  
  1.1256      SELECT COUNT(*) FROM ft020116jdbctableentity ;
  1.1257  
  1.1258 ---START--
  1.1259 - count
  1.1260 ---------
  1.1261 - 167273
  1.1262 -(1 row)
  1.1263 ---END--
  1.1264 +    >    count
  1.1265 +    >   --------
  1.1266 +    >    167273
  1.1267 +    >   (1 row)
  1.1268  
  1.1269      SELECT COUNT(*) FROM ft020116ogsadqpresourceentity ;
  1.1270  
  1.1271 ---START--
  1.1272 - count
  1.1273 --------
  1.1274 -     0
  1.1275 -(1 row)
  1.1276 ---END--
  1.1277 +    >    count
  1.1278 +    >   -------
  1.1279 +    >        0
  1.1280 +    >   (1 row)
  1.1281  
  1.1282      SELECT COUNT(*) FROM ft020116ogsaexecresourceentity ;
  1.1283  
  1.1284 ---START--
  1.1285 - count
  1.1286 ---------
  1.1287 - 165196
  1.1288 -(1 row)
  1.1289 ---END--
  1.1290 +    >    count
  1.1291 +    >   --------
  1.1292 +    >    165196
  1.1293 +    >   (1 row)
  1.1294  
  1.1295      SELECT COUNT(*) FROM ft020116ogsaivoaresourceentity ;
  1.1296  
  1.1297 ---START--
  1.1298 - count
  1.1299 --------
  1.1300 -     0
  1.1301 -(1 row)
  1.1302 ---END--
  1.1303 +    >    count
  1.1304 +    >   -------
  1.1305 +    >        0
  1.1306 +    >   (1 row)
  1.1307  
  1.1308      SELECT COUNT(*) FROM ft020116ogsajdbcresourceentity ;
  1.1309  
  1.1310 ---START--
  1.1311 - count
  1.1312 --------
  1.1313 -     4
  1.1314 -(1 row)
  1.1315 ---END--
  1.1316 +    >    count
  1.1317 +    >   -------
  1.1318 +    >        4
  1.1319 +    >   (1 row)
  1.1320  
  1.1321      SELECT COUNT(*) FROM ft020116ogsaserviceentity ;
  1.1322  
  1.1323 ---START--
  1.1324 - count
  1.1325 --------
  1.1326 -     1
  1.1327 -(1 row)
  1.1328 ---END--
  1.1329 +    >    count
  1.1330 +    >   -------
  1.1331 +    >        1
  1.1332 +    >   (1 row)
  1.1333  
  1.1334      SELECT COUNT(*) FROM ft020116operationentity ;
  1.1335  
  1.1336 ---START--
  1.1337 -  count
  1.1338 -----------
  1.1339 - 16020567
  1.1340 -(1 row)
  1.1341 ---END--
  1.1342 +    >     count
  1.1343 +    >   ----------
  1.1344 +    >    16020567
  1.1345 +    >   (1 row)
  1.1346  
  1.1347  
  1.1348  # -----------------------------------------------------
  1.1349 @@ -1013,20 +937,18 @@
  1.1350      # authmethodentity has way too many fields.
  1.1351      # It is just a link from [identity] to [operation] with [method]
  1.1352  
  1.1353 ---START--
  1.1354 -                 Table "public.ft020116authmethodentity"
  1.1355 -  Column   |            Type             | Collation | Nullable | Default
  1.1356 ------------+-----------------------------+-----------+----------+---------
  1.1357 - ident     | bigint                      |           | not null |
  1.1358 - created   | timestamp without time zone |           | not null |
  1.1359 - modified  | timestamp without time zone |           | not null |
  1.1360 - uidhi     | bigint                      |           | not null |
  1.1361 - uidlo     | bigint                      |           | not null |
  1.1362 - method    | character varying(255)      |           |          |
  1.1363 - owner     | bigint                      |           |          |
  1.1364 - identity  | bigint                      |           | not null |
  1.1365 - operation | bigint                      |           | not null |
  1.1366 ---END--
  1.1367 +    >                    Table "public.ft020116authmethodentity"
  1.1368 +    >     Column   |            Type             | Collation | Nullable | Default
  1.1369 +    >   -----------+-----------------------------+-----------+----------+---------
  1.1370 +    >    ident     | bigint                      |           | not null |
  1.1371 +    >    created   | timestamp without time zone |           | not null |
  1.1372 +    >    modified  | timestamp without time zone |           | not null |
  1.1373 +    >    uidhi     | bigint                      |           | not null |
  1.1374 +    >    uidlo     | bigint                      |           | not null |
  1.1375 +    >    method    | character varying(255)      |           |          |
  1.1376 +    >    owner     | bigint                      |           |          |
  1.1377 +    >    identity  | bigint                      |           | not null |
  1.1378 +    >    operation | bigint                      |           | not null |
  1.1379  
  1.1380      #
  1.1381      # Options for optimisation ... but nothing that would kill the service.
  1.1382 @@ -1044,12 +966,10 @@
  1.1383              --username "${tapschemauser:?}" \
  1.1384              --dbname   "${tapschemadata:?}"
  1.1385  
  1.1386 ---START--
  1.1387 -psql (10.6 (Debian 10.6-1.pgdg90+1))
  1.1388 -Type "help" for help.
  1.1389 -
  1.1390 -postgres=#
  1.1391 ---END--
  1.1392 +    >   psql (10.6 (Debian 10.6-1.pgdg90+1))
  1.1393 +    >   Type "help" for help.
  1.1394 +    >   
  1.1395 +    >   postgres=#
  1.1396  
  1.1397  # -----------------------------------------------------
  1.1398  # List the database tables.
  1.1399 @@ -1057,9 +977,7 @@
  1.1400  
  1.1401      \dt
  1.1402  
  1.1403 ---START--
  1.1404 -Did not find any relations.
  1.1405 ---END--
  1.1406 +    >   Did not find any relations.
  1.1407  
  1.1408  
  1.1409  # -----------------------------------------------------
  1.1410 @@ -1068,27 +986,23 @@
  1.1411  
  1.1412      grep 'tapschema' chain.properties
  1.1413  
  1.1414 ---START--
  1.1415 -tapschemadata=postgres
  1.1416 -tapschemauser=ieW7oomei7Laebuu9Ax7
  1.1417 -tapschemapass=vamee3puChoomahz9Iek
  1.1418 -tapschemahost=carolina
  1.1419 -tapschemaport=5432
  1.1420 -tapschematype=pgsql
  1.1421 -tapschemajdbc=tapschemajdbc
  1.1422 ---END--
  1.1423 +    >   tapschemadata=postgres
  1.1424 +    >   tapschemauser=ieW7oomei7Laebuu9Ax7
  1.1425 +    >   tapschemapass=vamee3puChoomahz9Iek
  1.1426 +    >   tapschemahost=carolina
  1.1427 +    >   tapschemaport=5432
  1.1428 +    >   tapschematype=pgsql
  1.1429 +    >   tapschemajdbc=tapschemajdbc
  1.1430  
  1.1431  
  1.1432      grep 'tapschema' firethorn.properties
  1.1433  
  1.1434 ---START--
  1.1435 -firethorn.tapschema.resource.name=tapschemajdbc
  1.1436 -firethorn.tapschema.database.name=postgres
  1.1437 -firethorn.tapschema.database.host=carolina
  1.1438 -firethorn.tapschema.database.port=5432
  1.1439 -firethorn.tapschema.database.user=ieW7oomei7Laebuu9Ax7
  1.1440 -firethorn.tapschema.database.pass=vamee3puChoomahz9Iek
  1.1441 ---END--
  1.1442 +    >   firethorn.tapschema.resource.name=tapschemajdbc
  1.1443 +    >   firethorn.tapschema.database.name=postgres
  1.1444 +    >   firethorn.tapschema.database.host=carolina
  1.1445 +    >   firethorn.tapschema.database.port=5432
  1.1446 +    >   firethorn.tapschema.database.user=ieW7oomei7Laebuu9Ax7
  1.1447 +    >   firethorn.tapschema.database.pass=vamee3puChoomahz9Iek
  1.1448  
  1.1449  
  1.1450  # -----------------------------------------------------
  1.1451 @@ -1107,15 +1021,13 @@
  1.1452  
  1.1453      docker-compose --file "docker-compose.yml" run firethorn-py
  1.1454  
  1.1455 ---START--
  1.1456 -Starting stevedore_bethany_1  ... done
  1.1457 -Starting stevedore_jarmila_1  ... done
  1.1458 -Starting stevedore_carolina_1 ... done
  1.1459 -Starting stevedore_gillian_1  ... done
  1.1460 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
  1.1461 -[GCC 5.4.0 20160609] on linux
  1.1462 -Type "help", "copyright", "credits" or "license" for more information.
  1.1463 ---END--
  1.1464 +    >   Starting stevedore_bethany_1  ... done
  1.1465 +    >   Starting stevedore_jarmila_1  ... done
  1.1466 +    >   Starting stevedore_carolina_1 ... done
  1.1467 +    >   Starting stevedore_gillian_1  ... done
  1.1468 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
  1.1469 +    >   [GCC 5.4.0 20160609] on linux
  1.1470 +    >   Type "help", "copyright", "credits" or "license" for more information.
  1.1471  
  1.1472  
  1.1473  # -----------------------------------------------------
  1.1474 @@ -1157,23 +1069,21 @@
  1.1475      )
  1.1476  
  1.1477  print(resource)
  1.1478 ---START--
  1.1479 -{
  1.1480 -  "name": "OSA ADQL resource",
  1.1481 -  "fullname": "OSA ADQL resource",
  1.1482 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/163",
  1.1483 -  "ident": "54",
  1.1484 -  "url": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1485 -  "modified": "2018-12-12T18:44:59.556",
  1.1486 -  "schemas": "http://tap.roe.ac.uk/firethorn/adql/resource/54/schemas/select",
  1.1487 -  "queries": "http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select",
  1.1488 -  "self": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1489 -  "created": "2018-12-12T18:44:59.555",
  1.1490 -  "vosi": "http://tap.roe.ac.uk/firethorn/adql/resource/54/vosi",
  1.1491 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
  1.1492 -  "text": null
  1.1493 -}
  1.1494 ---END--
  1.1495 +    >   {
  1.1496 +    >     "name": "OSA ADQL resource",
  1.1497 +    >     "fullname": "OSA ADQL resource",
  1.1498 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/163",
  1.1499 +    >     "ident": "54",
  1.1500 +    >     "url": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1501 +    >     "modified": "2018-12-12T18:44:59.556",
  1.1502 +    >     "schemas": "http://tap.roe.ac.uk/firethorn/adql/resource/54/schemas/select",
  1.1503 +    >     "queries": "http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select",
  1.1504 +    >     "self": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1505 +    >     "created": "2018-12-12T18:44:59.555",
  1.1506 +    >     "vosi": "http://tap.roe.ac.uk/firethorn/adql/resource/54/vosi",
  1.1507 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
  1.1508 +    >     "text": null
  1.1509 +    >   }
  1.1510  
  1.1511  #
  1.1512  # Create and run a query on the resource.
  1.1513 @@ -1189,143 +1099,137 @@
  1.1514      query_obj
  1.1515      )
  1.1516  
  1.1517 ---START--
  1.1518 -{
  1.1519 -  "columns": [],
  1.1520 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1521 -  "results": {
  1.1522 -    "state": "COMPLETED",
  1.1523 -    "count": 1000,
  1.1524 -    "formats": {
  1.1525 -      "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1526 -      "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1527 -    },
  1.1528 -    "table": "http://tap.roe.ac.uk/firethorn/adql/table/57699203"
  1.1529 -  },
  1.1530 -  "modified": "2019-05-09T03:57:24.377",
  1.1531 -  "tables": [],
  1.1532 -  "syntax": {
  1.1533 -    "friendly": null,
  1.1534 -    "status": "VALID",
  1.1535 -    "message": null
  1.1536 -  },
  1.1537 -  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
  1.1538 -  "ident": "57699002",
  1.1539 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
  1.1540 -  "history": [
  1.1541 -    {
  1.1542 -      "state": "COMPLETED",
  1.1543 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1544 -      "modified": "2019-05-09T03:57:24.327",
  1.1545 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1546 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698919",
  1.1547 -      "ident": "57699054",
  1.1548 -      "created": "2019-05-09T03:57:24.324",
  1.1549 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1550 -      "level": "INFO",
  1.1551 -      "message": null,
  1.1552 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1553 -    },
  1.1554 -    {
  1.1555 -      "state": "RUNNING",
  1.1556 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1557 -      "modified": "2019-05-09T03:57:23.988",
  1.1558 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1559 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698918",
  1.1560 -      "ident": "57699053",
  1.1561 -      "created": "2019-05-09T03:57:23.984",
  1.1562 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1563 -      "level": "INFO",
  1.1564 -      "message": null,
  1.1565 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1566 -    },
  1.1567 -    {
  1.1568 -      "state": "READY",
  1.1569 -      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1570 -      "modified": "2019-05-09T03:57:20.823",
  1.1571 -      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1572 -      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1573 -      "ident": "57699052",
  1.1574 -      "created": "2019-05-09T03:57:20.818",
  1.1575 -      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1576 -      "level": "INFO",
  1.1577 -      "message": "Executing query",
  1.1578 -      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1579 -    }
  1.1580 -  ],
  1.1581 -  "name": "XX_5Y47JQZFGVHTOAAAAFVJVPIFRQ",
  1.1582 -  "text": null,
  1.1583 -  "fields": [],
  1.1584 -  "url": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1585 -  "limits": {
  1.1586 -    "rows": 1000000,
  1.1587 -    "time": null,
  1.1588 -    "cells": null
  1.1589 -  },
  1.1590 -  "delays": {
  1.1591 -    "every": null,
  1.1592 -    "first": null,
  1.1593 -    "last": null
  1.1594 -  },
  1.1595 -  "adql": "SELECT TOP 1000 ra , dec\nFROM ATLASDR1.atlasSource",
  1.1596 -  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1597 -  "resources": [
  1.1598 -    "http://tap.roe.ac.uk/firethorn/jdbc/resource/53"
  1.1599 -  ],
  1.1600 -  "callback": "http://tap.roe.ac.uk:8081/firethorn/callback/57699002",
  1.1601 -  "mode": "DIRECT",
  1.1602 -  "created": "2019-05-09T03:57:20.140",
  1.1603 -  "osql": "SELECT TOP 1000 [ATLASDR1].[dbo].[atlasSource].[ra] AS [ra] , [ATLASDR1].[dbo].[atlasSource].[dec] AS [dec]\nFROM [ATLASDR1].[dbo].[atlasSource]",
  1.1604 -  "status": "COMPLETED",
  1.1605 -  "self": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1606 -  "workspace": "http://tap.roe.ac.uk/firethorn/adql/resource/54"
  1.1607 -}
  1.1608 ---END--
  1.1609 +    >   {
  1.1610 +    >     "columns": [],
  1.1611 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1612 +    >     "results": {
  1.1613 +    >       "state": "COMPLETED",
  1.1614 +    >       "count": 1000,
  1.1615 +    >       "formats": {
  1.1616 +    >         "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1617 +    >         "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1618 +    >       },
  1.1619 +    >       "table": "http://tap.roe.ac.uk/firethorn/adql/table/57699203"
  1.1620 +    >     },
  1.1621 +    >     "modified": "2019-05-09T03:57:24.377",
  1.1622 +    >     "tables": [],
  1.1623 +    >     "syntax": {
  1.1624 +    >       "friendly": null,
  1.1625 +    >       "status": "VALID",
  1.1626 +    >       "message": null
  1.1627 +    >     },
  1.1628 +    >     "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
  1.1629 +    >     "ident": "57699002",
  1.1630 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
  1.1631 +    >     "history": [
  1.1632 +    >       {
  1.1633 +    >         "state": "COMPLETED",
  1.1634 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1635 +    >         "modified": "2019-05-09T03:57:24.327",
  1.1636 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1637 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698919",
  1.1638 +    >         "ident": "57699054",
  1.1639 +    >         "created": "2019-05-09T03:57:24.324",
  1.1640 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1641 +    >         "level": "INFO",
  1.1642 +    >         "message": null,
  1.1643 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1644 +    >       },
  1.1645 +    >       {
  1.1646 +    >         "state": "RUNNING",
  1.1647 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1648 +    >         "modified": "2019-05-09T03:57:23.988",
  1.1649 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1650 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698918",
  1.1651 +    >         "ident": "57699053",
  1.1652 +    >         "created": "2019-05-09T03:57:23.984",
  1.1653 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1654 +    >         "level": "INFO",
  1.1655 +    >         "message": null,
  1.1656 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1657 +    >       },
  1.1658 +    >       {
  1.1659 +    >         "state": "READY",
  1.1660 +    >         "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1661 +    >         "modified": "2019-05-09T03:57:20.823",
  1.1662 +    >         "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1663 +    >         "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1664 +    >         "ident": "57699052",
  1.1665 +    >         "created": "2019-05-09T03:57:20.818",
  1.1666 +    >         "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1667 +    >         "level": "INFO",
  1.1668 +    >         "message": "Executing query",
  1.1669 +    >         "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1670 +    >       }
  1.1671 +    >     ],
  1.1672 +    >     "name": "XX_5Y47JQZFGVHTOAAAAFVJVPIFRQ",
  1.1673 +    >     "text": null,
  1.1674 +    >     "fields": [],
  1.1675 +    >     "url": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1676 +    >     "limits": {
  1.1677 +    >       "rows": 1000000,
  1.1678 +    >       "time": null,
  1.1679 +    >       "cells": null
  1.1680 +    >     },
  1.1681 +    >     "delays": {
  1.1682 +    >       "every": null,
  1.1683 +    >       "first": null,
  1.1684 +    >       "last": null
  1.1685 +    >     },
  1.1686 +    >     "adql": "SELECT TOP 1000 ra , dec\nFROM ATLASDR1.atlasSource",
  1.1687 +    >     "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1688 +    >     "resources": [
  1.1689 +    >       "http://tap.roe.ac.uk/firethorn/jdbc/resource/53"
  1.1690 +    >     ],
  1.1691 +    >     "callback": "http://tap.roe.ac.uk:8081/firethorn/callback/57699002",
  1.1692 +    >     "mode": "DIRECT",
  1.1693 +    >     "created": "2019-05-09T03:57:20.140",
  1.1694 +    >     "osql": "SELECT TOP 1000 [ATLASDR1].[dbo].[atlasSource].[ra] AS [ra] , [ATLASDR1].[dbo].[atlasSource].[dec] AS [dec]\nFROM [ATLASDR1].[dbo].[atlasSource]",
  1.1695 +    >     "status": "COMPLETED",
  1.1696 +    >     "self": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1697 +    >     "workspace": "http://tap.roe.ac.uk/firethorn/adql/resource/54"
  1.1698 +    >   }
  1.1699  
  1.1700  print(
  1.1701      query_obj.table()
  1.1702      )
  1.1703  
  1.1704 ---START--
  1.1705 -{
  1.1706 -  "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1707 -  "fullname": "temp.XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1708 -  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1709 -  "name": "XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1710 -  "url": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1711 -  "modified": "2019-05-09T03:57:20.903",
  1.1712 -  "ident": "57699203",
  1.1713 -  "depth": "PARTIAL",
  1.1714 -  "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1715 -  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/57699102",
  1.1716 -  "created": "2019-05-09T03:57:20.903",
  1.1717 -  "formats": {
  1.1718 -    "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1719 -    "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1720 -  },
  1.1721 -  "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1722 -  "columns": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/columns/select",
  1.1723 -  "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1724 -  "self": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1725 -  "query": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1726 -  "metadata": {
  1.1727 -    "adql": {
  1.1728 -      "count": 1000,
  1.1729 -      "status": "COMPLETED"
  1.1730 -    }
  1.1731 -  },
  1.1732 -  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
  1.1733 -  "text": null
  1.1734 -}
  1.1735 ---END--
  1.1736 +    >   {
  1.1737 +    >     "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1738 +    >     "fullname": "temp.XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1739 +    >     "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1740 +    >     "name": "XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1741 +    >     "url": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1742 +    >     "modified": "2019-05-09T03:57:20.903",
  1.1743 +    >     "ident": "57699203",
  1.1744 +    >     "depth": "PARTIAL",
  1.1745 +    >     "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1746 +    >     "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/57699102",
  1.1747 +    >     "created": "2019-05-09T03:57:20.903",
  1.1748 +    >     "formats": {
  1.1749 +    >       "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1750 +    >       "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1751 +    >     },
  1.1752 +    >     "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1753 +    >     "columns": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/columns/select",
  1.1754 +    >     "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1755 +    >     "self": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1756 +    >     "query": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1757 +    >     "metadata": {
  1.1758 +    >       "adql": {
  1.1759 +    >         "count": 1000,
  1.1760 +    >         "status": "COMPLETED"
  1.1761 +    >       }
  1.1762 +    >     },
  1.1763 +    >     "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
  1.1764 +    >     "text": null
  1.1765 +    >   }
  1.1766  
  1.1767  print(
  1.1768      query_obj.table().count()
  1.1769      )
  1.1770  
  1.1771 ---START--
  1.1772 -1000
  1.1773 ---END--
  1.1774 +    >   1000
  1.1775  
  1.1776  #
  1.1777  # Iterate the metadata tree
  1.1778 @@ -1351,50 +1255,48 @@
  1.1779          py_table = query_obj.table().as_astropy()
  1.1780          py_table.pprint()
  1.1781  
  1.1782 ---START--
  1.1783 -....
  1.1784 -....
  1.1785 -....
  1.1786 -table  [ATLASDR1][CurrentAstrometry][-1]
  1.1787 -Downloading http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable [Done]
  1.1788 -Traceback (most recent call last):
  1.1789 -  File "<stdin>", line 20, in <module>
  1.1790 -  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_table.py", line 97, in as_astropy
  1.1791 -    return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
  1.1792 -  File "/usr/local/lib/python3.5/dist-packages/astropy/table/table.py", line 2548, in read
  1.1793 -    out = io_registry.read(cls, *args, **kwargs)
  1.1794 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/registry.py", line 517, in read
  1.1795 -    data = reader(*args, **kwargs)
  1.1796 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/connect.py", line 73, in read_table_votable
  1.1797 -    input = parse(input, table_id=table_id)
  1.1798 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/table.py", line 137, in parse
  1.1799 -    config=config, pos=(1, 1)).parse(iterator, config)
  1.1800 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3390, in parse
  1.1801 -    iterator, tag, data, config, pos)
  1.1802 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3319, in _add_resource
  1.1803 -    resource.parse(self, iterator, config)
  1.1804 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3141, in parse
  1.1805 -    iterator, tag, data, config, pos)
  1.1806 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3092, in _add_table
  1.1807 -    table.parse(iterator, config)
  1.1808 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2382, in parse
  1.1809 -    iterator, colnumbers, fields, config)
  1.1810 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2497, in _parse_tabledata
  1.1811 -    vo_reraise(e, config, pos)
  1.1812 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1813 -    raise exc
  1.1814 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2494, in _parse_tabledata
  1.1815 -    fields[i].ID))
  1.1816 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1817 -    raise exc
  1.1818 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2488, in _parse_tabledata
  1.1819 -    data, config, pos)
  1.1820 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 543, in parse
  1.1821 -    value, mask = parse(x, config, pos)
  1.1822 -  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 815, in parse
  1.1823 -    value = int(value, 10)
  1.1824 -ValueError: http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable:1:21910: ValueError: invalid literal for int() with base 10: '0000000a' (in row 9, col 'column.57699287')
  1.1825 ---END--
  1.1826 +    >   ....
  1.1827 +    >   ....
  1.1828 +    >   ....
  1.1829 +    >   table  [ATLASDR1][CurrentAstrometry][-1]
  1.1830 +    >   Downloading http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable [Done]
  1.1831 +    >   Traceback (most recent call last):
  1.1832 +    >     File "<stdin>", line 20, in <module>
  1.1833 +    >     File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_table.py", line 97, in as_astropy
  1.1834 +    >       return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
  1.1835 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/table/table.py", line 2548, in read
  1.1836 +    >       out = io_registry.read(cls, *args, **kwargs)
  1.1837 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/registry.py", line 517, in read
  1.1838 +    >       data = reader(*args, **kwargs)
  1.1839 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/connect.py", line 73, in read_table_votable
  1.1840 +    >       input = parse(input, table_id=table_id)
  1.1841 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/table.py", line 137, in parse
  1.1842 +    >       config=config, pos=(1, 1)).parse(iterator, config)
  1.1843 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3390, in parse
  1.1844 +    >       iterator, tag, data, config, pos)
  1.1845 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3319, in _add_resource
  1.1846 +    >       resource.parse(self, iterator, config)
  1.1847 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3141, in parse
  1.1848 +    >       iterator, tag, data, config, pos)
  1.1849 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3092, in _add_table
  1.1850 +    >       table.parse(iterator, config)
  1.1851 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2382, in parse
  1.1852 +    >       iterator, colnumbers, fields, config)
  1.1853 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2497, in _parse_tabledata
  1.1854 +    >       vo_reraise(e, config, pos)
  1.1855 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1856 +    >       raise exc
  1.1857 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2494, in _parse_tabledata
  1.1858 +    >       fields[i].ID))
  1.1859 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1860 +    >       raise exc
  1.1861 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2488, in _parse_tabledata
  1.1862 +    >       data, config, pos)
  1.1863 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 543, in parse
  1.1864 +    >       value, mask = parse(x, config, pos)
  1.1865 +    >     File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 815, in parse
  1.1866 +    >       value = int(value, 10)
  1.1867 +    >   ValueError: http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable:1:21910: ValueError: invalid literal for int() with base 10: '0000000a' (in row 9, col 'column.57699287')
  1.1868  
  1.1869  
  1.1870  
  1.1871 @@ -1407,41 +1309,39 @@
  1.1872      # Requesting the large (171,174) list of queries for the main resource causes an Exception.
  1.1873      # http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select
  1.1874  
  1.1875 ---START--
  1.1876 -java.lang.OutOfMemoryError: GC overhead limit exceeded
  1.1877 -	org.hibernate.type.descriptor.sql.BigIntTypeDescriptor.getExtractor(BigIntTypeDescriptor.java:60)
  1.1878 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:261)
  1.1879 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:257)
  1.1880 -	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:247)
  1.1881 -	org.hibernate.type.AbstractStandardBasicType.hydrate(AbstractStandardBasicType.java:333)
  1.1882 -	org.hibernate.type.ComponentType.hydrate(ComponentType.java:663)
  1.1883 -	org.hibernate.persister.entity.AbstractEntityPersister.hydrate(AbstractEntityPersister.java:2972)
  1.1884 -	org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1746)
  1.1885 -	org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1672)
  1.1886 -	org.hibernate.loader.Loader.getRow(Loader.java:1561)
  1.1887 -	org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:731)
  1.1888 -	org.hibernate.loader.Loader.processResultSet(Loader.java:990)
  1.1889 -	org.hibernate.loader.Loader.doQuery(Loader.java:948)
  1.1890 -	org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:340)
  1.1891 -	org.hibernate.loader.Loader.doList(Loader.java:2689)
  1.1892 -	org.hibernate.loader.Loader.doList(Loader.java:2672)
  1.1893 -	org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2506)
  1.1894 -	org.hibernate.loader.Loader.list(Loader.java:2501)
  1.1895 -	org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:504)
  1.1896 -	org.hibernate.hql.internal.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:395)
  1.1897 -	org.hibernate.engine.query.spi.HQLQueryPlan.performList(HQLQueryPlan.java:221)
  1.1898 -	org.hibernate.internal.SessionImpl.list(SessionImpl.java:1508)
  1.1899 -	org.hibernate.query.internal.AbstractProducedQuery.doList(AbstractProducedQuery.java:1538)
  1.1900 -	org.hibernate.query.internal.AbstractProducedQuery.list(AbstractProducedQuery.java:1506)
  1.1901 -	uk.ac.roe.wfau.firethorn.entity.AbstractEntityFactory.list(AbstractEntityFactory.java:259)
  1.1902 -	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory.select(BlueQueryEntity.java:535)
  1.1903 -	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory$$FastClassBySpringCGLIB$$598856d8.invoke(<generated>)
  1.1904 -	org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
  1.1905 -	org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:746)
  1.1906 -	org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
  1.1907 -	org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$157/756591810.proceedWithInvocation(Unknown Source)
  1.1908 -	org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
  1.1909 ---END--
  1.1910 +    >   java.lang.OutOfMemoryError: GC overhead limit exceeded
  1.1911 +    >   	org.hibernate.type.descriptor.sql.BigIntTypeDescriptor.getExtractor(BigIntTypeDescriptor.java:60)
  1.1912 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:261)
  1.1913 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:257)
  1.1914 +    >   	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:247)
  1.1915 +    >   	org.hibernate.type.AbstractStandardBasicType.hydrate(AbstractStandardBasicType.java:333)
  1.1916 +    >   	org.hibernate.type.ComponentType.hydrate(ComponentType.java:663)
  1.1917 +    >   	org.hibernate.persister.entity.AbstractEntityPersister.hydrate(AbstractEntityPersister.java:2972)
  1.1918 +    >   	org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1746)
  1.1919 +    >   	org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1672)
  1.1920 +    >   	org.hibernate.loader.Loader.getRow(Loader.java:1561)
  1.1921 +    >   	org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:731)
  1.1922 +    >   	org.hibernate.loader.Loader.processResultSet(Loader.java:990)
  1.1923 +    >   	org.hibernate.loader.Loader.doQuery(Loader.java:948)
  1.1924 +    >   	org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:340)
  1.1925 +    >   	org.hibernate.loader.Loader.doList(Loader.java:2689)
  1.1926 +    >   	org.hibernate.loader.Loader.doList(Loader.java:2672)
  1.1927 +    >   	org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2506)
  1.1928 +    >   	org.hibernate.loader.Loader.list(Loader.java:2501)
  1.1929 +    >   	org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:504)
  1.1930 +    >   	org.hibernate.hql.internal.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:395)
  1.1931 +    >   	org.hibernate.engine.query.spi.HQLQueryPlan.performList(HQLQueryPlan.java:221)
  1.1932 +    >   	org.hibernate.internal.SessionImpl.list(SessionImpl.java:1508)
  1.1933 +    >   	org.hibernate.query.internal.AbstractProducedQuery.doList(AbstractProducedQuery.java:1538)
  1.1934 +    >   	org.hibernate.query.internal.AbstractProducedQuery.list(AbstractProducedQuery.java:1506)
  1.1935 +    >   	uk.ac.roe.wfau.firethorn.entity.AbstractEntityFactory.list(AbstractEntityFactory.java:259)
  1.1936 +    >   	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory.select(BlueQueryEntity.java:535)
  1.1937 +    >   	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory$$FastClassBySpringCGLIB$$598856d8.invoke(<generated>)
  1.1938 +    >   	org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
  1.1939 +    >   	org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:746)
  1.1940 +    >   	org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
  1.1941 +    >   	org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$157/756591810.proceedWithInvocation(Unknown Source)
  1.1942 +    >   	org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
  1.1943  
  1.1944  
  1.1945      #
  1.1946 @@ -1450,9 +1350,7 @@
  1.1947  
  1.1948      \dt
  1.1949  
  1.1950 ---START--
  1.1951 -Did not find any relations.
  1.1952 ---END--
  1.1953 +    >   Did not find any relations.
  1.1954  
  1.1955      #
  1.1956      # But these resources all work.
     2.1 --- a/doc/notes/zrq/20190508-02-vmdown-debug.txt	Mon Jun 03 04:24:03 2019 +0100
     2.2 +++ b/doc/notes/zrq/20190508-02-vmdown-debug.txt	Mon Jun 03 04:46:00 2019 +0100
     2.3 @@ -35,15 +35,13 @@
     2.4      source "${HOME:?}/chain.properties"
     2.5      docker-compose --file "docker-compose.yml" run firethorn-py
     2.6  
     2.7 ---START--
     2.8 -Starting stevedore_bethany_1  ... done
     2.9 -Starting stevedore_jarmila_1  ... done
    2.10 -Starting stevedore_carolina_1 ... done
    2.11 -Starting stevedore_gillian_1  ... done
    2.12 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    2.13 -[GCC 5.4.0 20160609] on linux
    2.14 -Type "help", "copyright", "credits" or "license" for more information.
    2.15 ---END--
    2.16 +    >   Starting stevedore_bethany_1  ... done
    2.17 +    >   Starting stevedore_jarmila_1  ... done
    2.18 +    >   Starting stevedore_carolina_1 ... done
    2.19 +    >   Starting stevedore_gillian_1  ... done
    2.20 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    2.21 +    >   [GCC 5.4.0 20160609] on linux
    2.22 +    >   Type "help", "copyright", "credits" or "license" for more information.
    2.23  
    2.24  # -----------------------------------------------------
    2.25  # Check the key resources.
    2.26 @@ -88,25 +86,21 @@
    2.27  
    2.28      free -h
    2.29  
    2.30 ---START--
    2.31 -              total        used        free      shared  buff/cache   available
    2.32 -Mem:           3.9G        1.8G        245M        156M        1.8G        1.7G
    2.33 -Swap:          1.0G         71M        952M
    2.34 ---END--
    2.35 +    >                 total        used        free      shared  buff/cache   available
    2.36 +    >   Mem:           3.9G        1.8G        245M        156M        1.8G        1.7G
    2.37 +    >   Swap:          1.0G         71M        952M
    2.38  
    2.39      df -h
    2.40  
    2.41 ---START--
    2.42 -Filesystem      Size  Used Avail Use% Mounted on
    2.43 -devtmpfs        2.0G     0  2.0G   0% /dev
    2.44 -tmpfs           2.0G     0  2.0G   0% /dev/shm
    2.45 -tmpfs           2.0G  844K  2.0G   1% /run
    2.46 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
    2.47 -/dev/vda3        31G   23G  7.7G  75% /
    2.48 -tmpfs           2.0G  4.0K  2.0G   1% /tmp
    2.49 -/dev/vda1       240M   89M  135M  40% /boot
    2.50 -tmpfs           395M     0  395M   0% /run/user/1001
    2.51 ---END--
    2.52 +    >   Filesystem      Size  Used Avail Use% Mounted on
    2.53 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
    2.54 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
    2.55 +    >   tmpfs           2.0G  844K  2.0G   1% /run
    2.56 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
    2.57 +    >   /dev/vda3        31G   23G  7.7G  75% /
    2.58 +    >   tmpfs           2.0G  4.0K  2.0G   1% /tmp
    2.59 +    >   /dev/vda1       240M   89M  135M  40% /boot
    2.60 +    >   tmpfs           395M     0  395M   0% /run/user/1001
    2.61  
    2.62  
    2.63  # -----------------------------------------------------
    2.64 @@ -201,39 +195,31 @@
    2.65  
    2.66      date
    2.67  
    2.68 ---START--
    2.69 -Thu  9 May 22:45:35 BST 2019
    2.70 ---END--
    2.71 +    >   Thu  9 May 22:45:35 BST 2019
    2.72  
    2.73  free
    2.74  
    2.75 ---START--
    2.76 -              total        used        free      shared  buff/cache   available
    2.77 -Mem:        4038224     1776084      220624      159980     2041516     1871952
    2.78 -Swap:       1048572       88320      960252
    2.79 ---END--
    2.80 +    >                 total        used        free      shared  buff/cache   available
    2.81 +    >   Mem:        4038224     1776084      220624      159980     2041516     1871952
    2.82 +    >   Swap:       1048572       88320      960252
    2.83  
    2.84  free -h
    2.85  
    2.86 ---START--
    2.87 -              total        used        free      shared  buff/cache   available
    2.88 -Mem:           3.9G        1.7G        172M        156M        2.0G        1.8G
    2.89 -Swap:          1.0G         86M        937M
    2.90 ---END--
    2.91 +    >                 total        used        free      shared  buff/cache   available
    2.92 +    >   Mem:           3.9G        1.7G        172M        156M        2.0G        1.8G
    2.93 +    >   Swap:          1.0G         86M        937M
    2.94  
    2.95      df -h
    2.96  
    2.97 ---START--
    2.98 -Filesystem      Size  Used Avail Use% Mounted on
    2.99 -devtmpfs        2.0G     0  2.0G   0% /dev
   2.100 -tmpfs           2.0G     0  2.0G   0% /dev/shm
   2.101 -tmpfs           2.0G  828K  2.0G   1% /run
   2.102 -tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   2.103 -/dev/vda3        31G   23G  7.7G  75% /
   2.104 -tmpfs           2.0G  4.0K  2.0G   1% /tmp
   2.105 -/dev/vda1       240M   89M  135M  40% /boot
   2.106 -tmpfs           395M     0  395M   0% /run/user/1001
   2.107 ---END--
   2.108 +    >   Filesystem      Size  Used Avail Use% Mounted on
   2.109 +    >   devtmpfs        2.0G     0  2.0G   0% /dev
   2.110 +    >   tmpfs           2.0G     0  2.0G   0% /dev/shm
   2.111 +    >   tmpfs           2.0G  828K  2.0G   1% /run
   2.112 +    >   tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   2.113 +    >   /dev/vda3        31G   23G  7.7G  75% /
   2.114 +    >   tmpfs           2.0G  4.0K  2.0G   1% /tmp
   2.115 +    >   /dev/vda1       240M   89M  135M  40% /boot
   2.116 +    >   tmpfs           395M     0  395M   0% /run/user/1001
   2.117  
   2.118      #
   2.119      # Memory not an issue (1.8G available).
   2.120 @@ -242,20 +228,18 @@
   2.121  
   2.122      docker ps -a
   2.123  
   2.124 ---START--
   2.125 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   2.126 -058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                18 hours ago        Exited (0) 17 hours ago                              stevedore_firethorn-py_run_7
   2.127 -e6deacef850b        firethorn/firethorn-py:2.1.28   "python3"                26 hours ago        Exited (137) 18 hours ago                            stevedore_firethorn-py_run_6
   2.128 -16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                13 days ago         Exited (137) 13 days ago                             stevedore_firethorn-py_run_5
   2.129 -2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 13 days ago                             stevedore_firethorn-py_run_4
   2.130 -659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                              stevedore_firethorn-py_run_3
   2.131 -cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                            stevedore_firethorn-py_run_2
   2.132 -ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                            stevedore_firethorn-py_run_1
   2.133 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (unhealthy)     0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   2.134 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   2.135 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_carolina_1
   2.136 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_bethany_1
   2.137 ---END--
   2.138 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   2.139 +    >   058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                18 hours ago        Exited (0) 17 hours ago                              stevedore_firethorn-py_run_7
   2.140 +    >   e6deacef850b        firethorn/firethorn-py:2.1.28   "python3"                26 hours ago        Exited (137) 18 hours ago                            stevedore_firethorn-py_run_6
   2.141 +    >   16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                13 days ago         Exited (137) 13 days ago                             stevedore_firethorn-py_run_5
   2.142 +    >   2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 13 days ago                             stevedore_firethorn-py_run_4
   2.143 +    >   659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                              stevedore_firethorn-py_run_3
   2.144 +    >   cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                            stevedore_firethorn-py_run_2
   2.145 +    >   ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                            stevedore_firethorn-py_run_1
   2.146 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (unhealthy)     0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   2.147 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   2.148 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_carolina_1
   2.149 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_bethany_1
   2.150  
   2.151      #
   2.152      # Tomcat is unhealthy because it is not responding to HTTP requests for system status.
   2.153 @@ -270,82 +254,77 @@
   2.154  
   2.155          tail -n 100 logs/firethorn-debug.log
   2.156  
   2.157 ---START--
   2.158 -2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [BaseProtector] isAdmin(Identity) [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   2.159 -2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] admin()
   2.160 -2019-05-09 21:48:00,018 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] find (Community, String) [Jo9keiRu7moo6ooNee1s]
   2.161 -2019-05-09 21:48:00,019 DEBUG [FireThornTaskScheduler-84] [IdentityEntity]   found [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   2.162 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [UserDataCleaner]   schema [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.163 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] tables() for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.164 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scan for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.165 -2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scantest for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.166 -....
   2.167 -....
   2.168 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] factory()
   2.169 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   2.170 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.171 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.172 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   2.173 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   2.174 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   2.175 -2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:50:00.017Z]
   2.176 ---END--
   2.177 +    >   2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [BaseProtector] isAdmin(Identity) [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   2.178 +    >   2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] admin()
   2.179 +    >   2019-05-09 21:48:00,018 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] find (Community, String) [Jo9keiRu7moo6ooNee1s]
   2.180 +    >   2019-05-09 21:48:00,019 DEBUG [FireThornTaskScheduler-84] [IdentityEntity]   found [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   2.181 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [UserDataCleaner]   schema [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.182 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] tables() for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.183 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scan for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.184 +    >   2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scantest for [252][FirethornUserdataTAP20181212LIVE.dbo]
   2.185 +    >   ....
   2.186 +    >   ....
   2.187 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] factory()
   2.188 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   2.189 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.190 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.191 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   2.192 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   2.193 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   2.194 +    >   2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:50:00.017Z]
   2.195  
   2.196  
   2.197          tail -n 200 logs/firethorn-trace.log
   2.198  
   2.199 ---START--
   2.200 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   2.201 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.202 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.203 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:28:00.081Z]
   2.204 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   2.205 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   2.206 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   2.207 -2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Authentications, Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   2.208 -....
   2.209 -....
   2.210 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory] scanperiod()
   2.211 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.212 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.213 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   2.214 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] scanperiod [PT25H]
   2.215 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prev scan is recent - skipping
   2.216 -2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:52:00.014Z]
   2.217 -2019-05-09 21:52:00,194 TRACE [FireThornTaskScheduler-93] [UserDataCleaner] -- DONE --
   2.218 -
   2.219 -    #
   2.220 -    # Need to setup a ssh tunnel for the Java debugger to see if Threads are an issue.
   2.221 -    #
   2.222 -
   2.223 -    #
   2.224 -    # Reboot the VM ..
   2.225 -    #
   2.226 -
   2.227 -# -----------------------------------------------------
   2.228 -# Increase the virtual machine memory.
   2.229 -#[user@trop02]
   2.230 -
   2.231 -    vmname=Araybwyn
   2.232 -
   2.233 -    source ${HOME}/libvirt.settings
   2.234 -
   2.235 -    virsh -c ${connection:?} \
   2.236 -        dumpxml "${vmname}"
   2.237 -
   2.238 ---START--
   2.239 -<domain type='kvm' id='9'>
   2.240 -  <name>Araybwyn</name>
   2.241 -  <uuid>a65ad4ca-0662-4c43-a32c-91e4180d87bc</uuid>
   2.242 -  <memory unit='KiB'>4194304</memory>
   2.243 -  <currentMemory unit='KiB'>4194304</currentMemory>
   2.244 -  <vcpu placement='static'>4</vcpu>
   2.245 -  <resource>
   2.246 -    <partition>/machine</partition>
   2.247 -  </resource>
   2.248 -  ....
   2.249 -</domain>
   2.250 ---END--
   2.251 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   2.252 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.253 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.254 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:28:00.081Z]
   2.255 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   2.256 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   2.257 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   2.258 +    >   2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Authentications, Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   2.259 +    >   ....
   2.260 +    >   ....
   2.261 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory] scanperiod()
   2.262 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory]   value [PT25H]
   2.263 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] Factory scanperiod  [PT25H]
   2.264 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   2.265 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] scanperiod [PT25H]
   2.266 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prev scan is recent - skipping
   2.267 +    >   2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:52:00.014Z]
   2.268 +    >   2019-05-09 21:52:00,194 TRACE [FireThornTaskScheduler-93] [UserDataCleaner] -- DONE --
   2.269 +    >   
   2.270 +    >       #
   2.271 +    >       # Need to setup a ssh tunnel for the Java debugger to see if Threads are an issue.
   2.272 +    >       #
   2.273 +    >   
   2.274 +    >       #
   2.275 +    >       # Reboot the VM ..
   2.276 +    >       #
   2.277 +    >   
   2.278 +    >   # -----------------------------------------------------
   2.279 +    >   # Increase the virtual machine memory.
   2.280 +    >   #[user@trop02]
   2.281 +    >   
   2.282 +    >       vmname=Araybwyn
   2.283 +    >   
   2.284 +    >       source ${HOME}/libvirt.settings
   2.285 +    >   
   2.286 +    >       virsh -c ${connection:?} \
   2.287 +    >           dumpxml "${vmname}"
   2.288 +    >   
   2.289 +    >   <domain type='kvm' id='9'>
   2.290 +    >     <name>Araybwyn</name>
   2.291 +    >     <uuid>a65ad4ca-0662-4c43-a32c-91e4180d87bc</uuid>
   2.292 +    >     <memory unit='KiB'>4194304</memory>
   2.293 +    >     <currentMemory unit='KiB'>4194304</currentMemory>
   2.294 +    >     <vcpu placement='static'>4</vcpu>
   2.295 +    >     <resource>
   2.296 +    >       <partition>/machine</partition>
   2.297 +    >     </resource>
   2.298 +    >     ....
   2.299 +    >   </domain>
   2.300  
   2.301      virsh -c ${connection:?} \
   2.302          shutdown "${vmname}"
   2.303 @@ -371,15 +350,13 @@
   2.304  
   2.305      docker ps -a
   2.306  
   2.307 ---START--
   2.308 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS               NAMES
   2.309 -058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                19 hours ago        Exited (0) 18 hours ago                          stevedore_firethorn-py_run_7
   2.310 -....
   2.311 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_gillian_1
   2.312 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_jarmila_1
   2.313 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_carolina_1
   2.314 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_bethany_1
   2.315 ---END--
   2.316 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS               NAMES
   2.317 +    >   058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                19 hours ago        Exited (0) 18 hours ago                          stevedore_firethorn-py_run_7
   2.318 +    >   ....
   2.319 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_gillian_1
   2.320 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_jarmila_1
   2.321 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_carolina_1
   2.322 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_bethany_1
   2.323  
   2.324  # -----------------------------------------------------
   2.325  # Restart our services..
   2.326 @@ -390,16 +367,14 @@
   2.327      source "${HOME:?}/chain.properties"
   2.328      docker-compose --file "docker-compose.yml" run firethorn-py
   2.329  
   2.330 ---START--
   2.331 -Starting stevedore_carolina_1 ... done
   2.332 -Starting stevedore_jarmila_1  ... done
   2.333 -Starting stevedore_bethany_1  ... done
   2.334 -Starting stevedore_gillian_1  ... done
   2.335 -Python 3.5.2 (default, Nov 12 2018, 13:43:14)
   2.336 -[GCC 5.4.0 20160609] on linux
   2.337 -Type "help", "copyright", "credits" or "license" for more information.
   2.338 ->>>
   2.339 ---END--
   2.340 +    >   Starting stevedore_carolina_1 ... done
   2.341 +    >   Starting stevedore_jarmila_1  ... done
   2.342 +    >   Starting stevedore_bethany_1  ... done
   2.343 +    >   Starting stevedore_gillian_1  ... done
   2.344 +    >   Python 3.5.2 (default, Nov 12 2018, 13:43:14)
   2.345 +    >   [GCC 5.4.0 20160609] on linux
   2.346 +    >   Type "help", "copyright", "credits" or "license" for more information.
   2.347 +    >   >>>
   2.348  
   2.349  
   2.350  
   2.351 @@ -412,39 +387,31 @@
   2.352  
   2.353      date
   2.354  
   2.355 ---START--
   2.356 -Fri 10 May 10:59:06 BST 2019
   2.357 ---END--
   2.358 +    >   Fri 10 May 10:59:06 BST 2019
   2.359  
   2.360  free
   2.361  
   2.362 ---START--
   2.363 -              total        used        free      shared  buff/cache   available
   2.364 -Mem:        7599060     2001516     3392964      159900     2204580     5195720
   2.365 -Swap:       1048572           0     1048572
   2.366 ---END--
   2.367 +    >                 total        used        free      shared  buff/cache   available
   2.368 +    >   Mem:        7599060     2001516     3392964      159900     2204580     5195720
   2.369 +    >   Swap:       1048572           0     1048572
   2.370  
   2.371  free -h
   2.372  
   2.373 ---START--
   2.374 -              total        used        free      shared  buff/cache   available
   2.375 -Mem:           7.2G        1.9G        3.2G        156M        2.1G        5.0G
   2.376 -Swap:          1.0G          0B        1.0G
   2.377 ---END--
   2.378 +    >                 total        used        free      shared  buff/cache   available
   2.379 +    >   Mem:           7.2G        1.9G        3.2G        156M        2.1G        5.0G
   2.380 +    >   Swap:          1.0G          0B        1.0G
   2.381  
   2.382      df -h
   2.383  
   2.384 ---START--
   2.385 -Filesystem      Size  Used Avail Use% Mounted on
   2.386 -devtmpfs        3.7G     0  3.7G   0% /dev
   2.387 -tmpfs           3.7G     0  3.7G   0% /dev/shm
   2.388 -tmpfs           3.7G  780K  3.7G   1% /run
   2.389 -tmpfs           3.7G     0  3.7G   0% /sys/fs/cgroup
   2.390 -/dev/vda3        31G   23G  7.6G  75% /
   2.391 -tmpfs           3.7G  4.0K  3.7G   1% /tmp
   2.392 -/dev/vda1       240M   89M  135M  40% /boot
   2.393 -tmpfs           743M     0  743M   0% /run/user/1001
   2.394 ---END--
   2.395 +    >   Filesystem      Size  Used Avail Use% Mounted on
   2.396 +    >   devtmpfs        3.7G     0  3.7G   0% /dev
   2.397 +    >   tmpfs           3.7G     0  3.7G   0% /dev/shm
   2.398 +    >   tmpfs           3.7G  780K  3.7G   1% /run
   2.399 +    >   tmpfs           3.7G     0  3.7G   0% /sys/fs/cgroup
   2.400 +    >   /dev/vda3        31G   23G  7.6G  75% /
   2.401 +    >   tmpfs           3.7G  4.0K  3.7G   1% /tmp
   2.402 +    >   /dev/vda1       240M   89M  135M  40% /boot
   2.403 +    >   tmpfs           743M     0  743M   0% /run/user/1001
   2.404  
   2.405      #
   2.406      # Memory not an issue (5G available).
   2.407 @@ -453,14 +420,12 @@
   2.408  
   2.409      docker ps -a
   2.410  
   2.411 ---START--
   2.412 -CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   2.413 -....
   2.414 -513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   2.415 -f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   2.416 -7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_carolina_1
   2.417 -80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_bethany_1
   2.418 ---END--
   2.419 +    >   CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   2.420 +    >   ....
   2.421 +    >   513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   2.422 +    >   f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 11 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   2.423 +    >   7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_carolina_1
   2.424 +    >   80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 11 hours                 5432/tcp                 stevedore_bethany_1
   2.425  
   2.426  # -----------------------------------------------------
   2.427  # Check the system status.
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/doc/notes/zrq/20190601-01-hg-merge.txt	Mon Jun 03 04:46:00 2019 +0100
     3.3 @@ -0,0 +1,577 @@
     3.4 +#
     3.5 +# <meta:header>
     3.6 +#   <meta:licence>
     3.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     3.8 +#
     3.9 +#     This information is free software: you can redistribute it and/or modify
    3.10 +#     it under the terms of the GNU General Public License as published by
    3.11 +#     the Free Software Foundation, either version 3 of the License, or
    3.12 +#     (at your option) any later version.
    3.13 +#
    3.14 +#     This information is distributed in the hope that it will be useful,
    3.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    3.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    3.17 +#     GNU General Public License for more details.
    3.18 +#
    3.19 +#     You should have received a copy of the GNU General Public License
    3.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    3.21 +#   </meta:licence>
    3.22 +# </meta:header>
    3.23 +#
    3.24 +#
    3.25 +
    3.26 +# -----------------------------------------------------
    3.27 +# Create a new VM.
    3.28 +#[user@trop01]
    3.29 +
    3.30 +    createvm
    3.31 +
    3.32 +    >   INFO : Node name [Astoalith]
    3.33 +    >   INFO : Base name [fedora-28-docker-base-20180708.qcow]
    3.34 +    >   INFO : Base path [/var/lib/libvirt/images/base/fedora-28-docker-base-20180708.qcow]
    3.35 +    >   INFO : Disc name [Astoalith.qcow]
    3.36 +    >   INFO : Disc size [16GiB]
    3.37 +
    3.38 +    vmname=Astoalith
    3.39 +
    3.40 +
    3.41 +# -----------------------------------------------------
    3.42 +# Define a hosts lookup function.
    3.43 +# https://askubuntu.com/questions/627906/why-is-my-etc-hosts-file-not-queried-when-nslookup-tries-to-resolve-an-address#comment1536517_627909
    3.44 +#[user@trop01]
    3.45 +
    3.46 +    getipv4()
    3.47 +        {
    3.48 +        getent hosts "${1:?}" | cut -d ' ' -f 1
    3.49 +        }
    3.50 +
    3.51 +
    3.52 +#---------------------------------------------------------------------
    3.53 +# Update the ssh keys for the VM.
    3.54 +#[user@trop01]
    3.55 +
    3.56 +    # Remove the current key.
    3.57 +    ssh-keygen -q -R "${vmname:?}"
    3.58 +
    3.59 +    # Add the host key(s) to known_hosts
    3.60 +    ssh-keyscan "${vmname:?}" >> "${HOME}/.ssh/known_hosts"
    3.61 +
    3.62 +    # Add the IP address key(s) to known_hosts
    3.63 +    ssh-keyscan -t ecdsa $(getipv4 "${vmname:?}") >> "${HOME}/.ssh/known_hosts"
    3.64 +
    3.65 +
    3.66 +# -----------------------------------------------------
    3.67 +# Login as Stevedore
    3.68 +#[user@trop]
    3.69 +
    3.70 +    ssh "${vmname:?}"
    3.71 +
    3.72 +
    3.73 +# -----------------------------------------------------
    3.74 +# Install our secret function.
    3.75 +#[user@virtual]
    3.76 +
    3.77 +    mkdir "${HOME:?}/bin"
    3.78 +    cat > "${HOME:?}/bin/secret" << 'EOF'
    3.79 +#!/bin/sh
    3.80 +ssh -n \
    3.81 +    'dave@shepseskaf.roe.ac.uk' \
    3.82 +    "bin/secret '${1}'"
    3.83 +EOF
    3.84 +
    3.85 +    chmod u+x "${HOME:?}/bin/secret"
    3.86 +    secret 'frog'
    3.87 +
    3.88 +
    3.89 +# -----------------------------------------------------
    3.90 +# Download our builder compose file
    3.91 +#[user@virtual]
    3.92 +
    3.93 +    wget -O builder.yml \
    3.94 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
    3.95 +
    3.96 +
    3.97 +# -----------------------------------------------------
    3.98 +# Set the target branch
    3.99 +#[user@virtual]
   3.100 +
   3.101 +    branch=2.1.23-zrq-update-depends
   3.102 +    branch=2.1.24-zrq-userdata-drop
   3.103 +    branch=2.1.25-zrq-tap-controller
   3.104 +    branch=2.1.31-stv-cleanup
   3.105 +
   3.106 +
   3.107 +# -----------------------------------------------------
   3.108 +# Run our builder.
   3.109 +#[user@virtual]
   3.110 +
   3.111 +    export branch
   3.112 +    export secretsh=$(which secret)
   3.113 +
   3.114 +    docker-compose \
   3.115 +        --file "builder.yml" \
   3.116 +        run \
   3.117 +            builder
   3.118 +
   3.119 +    # -----------------------------------------------------
   3.120 +    # Test our secret function.
   3.121 +    #[root@builder]
   3.122 +
   3.123 +        secret 'frog'
   3.124 +
   3.125 +    # -----------------------------------------------------
   3.126 +    # Initialise our paths.
   3.127 +    #[root@builder]
   3.128 +
   3.129 +        PATH=${PATH}:/builder/bin
   3.130 +
   3.131 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   3.132 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   3.133 +
   3.134 +        export FIRETHORN_HOME
   3.135 +        export FIRETHORN_CODE
   3.136 +
   3.137 +    # -----------------------------------------------------
   3.138 +    # Checkout a copy of our source code.
   3.139 +    #[root@builder]
   3.140 +
   3.141 +        02.01-checkout.sh
   3.142 +
   3.143 +    # -----------------------------------------------------
   3.144 +    # Merge our branch into main.
   3.145 +    #[root@builder]
   3.146 +
   3.147 +        07.01-merge-start.sh
   3.148 +
   3.149 +    # -----------------------------------------------------
   3.150 +    # Update our version number.
   3.151 +    #[root@builder]
   3.152 +
   3.153 +        07.02-version-step.sh
   3.154 +
   3.155 +    # -----------------------------------------------------
   3.156 +    # Build our base images.
   3.157 +    #[root@builder]
   3.158 +
   3.159 +        04.01-buildbase.sh
   3.160 +
   3.161 +    # -----------------------------------------------------
   3.162 +    # Tag this version of builder as latest.
   3.163 +    #[root@builder]
   3.164 +
   3.165 +        source "${FIRETHORN_HOME}/merge.settings"
   3.166 +
   3.167 +        docker tag \
   3.168 +            "firethorn/builder:${newversion}" "firethorn/builder:latest"
   3.169 +
   3.170 +    # -----------------------------------------------------
   3.171 +    # Exit this version of builder.
   3.172 +    #[root@builder]
   3.173 +
   3.174 +        exit
   3.175 +
   3.176 +# -----------------------------------------------------
   3.177 +# Run a new instance of the builder.
   3.178 +#[user@virtual]
   3.179 +
   3.180 +    export branch
   3.181 +    export secretsh=$(which secret)
   3.182 +
   3.183 +    docker-compose \
   3.184 +        --file "builder.yml" \
   3.185 +        run \
   3.186 +            builder
   3.187 +
   3.188 +    # -----------------------------------------------------
   3.189 +    # Test our secret function.
   3.190 +    #[root@builder]
   3.191 +
   3.192 +        secret 'frog'
   3.193 +
   3.194 +    # -----------------------------------------------------
   3.195 +    # Initialise our paths.
   3.196 +    #[root@builder]
   3.197 +
   3.198 +        PATH=${PATH}:/builder/bin
   3.199 +
   3.200 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   3.201 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   3.202 +
   3.203 +        export FIRETHORN_HOME
   3.204 +        export FIRETHORN_CODE
   3.205 +
   3.206 +    # -----------------------------------------------------
   3.207 +    # Build our base images.
   3.208 +    #[root@builder]
   3.209 +
   3.210 +        04.01-buildbase.sh
   3.211 +
   3.212 +    # -----------------------------------------------------
   3.213 +    # Compile our Java code.
   3.214 +    #[root@builder]
   3.215 +
   3.216 +        05.01-javamaven.sh
   3.217 +
   3.218 +    # -----------------------------------------------------
   3.219 +    # Build our Java containers.
   3.220 +    #[root@builder]
   3.221 +
   3.222 +        05.02-javadocker.sh
   3.223 +
   3.224 +    # -----------------------------------------------------
   3.225 +    # Fetch our Python code.
   3.226 +    #[root@builder]
   3.227 +
   3.228 +        echo "Fetching client source"
   3.229 +
   3.230 +        ftpysrc='/var/local/build/client'
   3.231 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
   3.232 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
   3.233 +
   3.234 +        if [  -e "${ftpysrc:?}" ]
   3.235 +        then
   3.236 +            pushd "${ftpysrc:?}"
   3.237 +
   3.238 +                echo "Updating Python code"
   3.239 +                git pull
   3.240 +
   3.241 +            popd
   3.242 +        else
   3.243 +            pushd "$(dirname ${ftpysrc:?})"
   3.244 +
   3.245 +                echo "Cloning Python code from [${ftpygit:?}]"
   3.246 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
   3.247 +
   3.248 +            popd
   3.249 +        fi
   3.250 +
   3.251 +    # -----------------------------------------------------
   3.252 +    # Delete the embedded metadata.
   3.253 +    #[root@builder]
   3.254 +
   3.255 +        pushd "${ftpysrc:?}"
   3.256 +
   3.257 +            du -h  firethorn/meta
   3.258 +            rm -rf firethorn/meta
   3.259 +
   3.260 +        popd
   3.261 +
   3.262 +    # -----------------------------------------------------
   3.263 +    # Build our Python container.
   3.264 +    #[root@builder]
   3.265 +
   3.266 +        echo "Building client image"
   3.267 +
   3.268 +        pushd "${FIRETHORN_CODE:?}"
   3.269 +
   3.270 +            source 'bin/util.sh'
   3.271 +            export buildtag=$(getbuildtag)
   3.272 +            export buildsrc=$(pwd)
   3.273 +            export ftpysrc
   3.274 +
   3.275 +            docker-compose \
   3.276 +                --file docker/compose/client/firethorn-py.yml \
   3.277 +                build
   3.278 +
   3.279 +        popd
   3.280 +
   3.281 +# -----------------------------------------------------
   3.282 +# Run our Python tests, 20190601-02-merge-tests.txt
   3.283 +# -----------------------------------------------------
   3.284 +
   3.285 +    # -----------------------------------------------------
   3.286 +    # Update our Mercurial config.
   3.287 +    #[root@builder]
   3.288 +
   3.289 +        07.02-mercurial-user.sh
   3.290 +
   3.291 +    >   /var/local/build/firethorn /
   3.292 +    >   /
   3.293 +
   3.294 +
   3.295 +    # -----------------------------------------------------
   3.296 +    # Commit our merged code.
   3.297 +    #[root@builder]
   3.298 +
   3.299 +        07.03-merge-commit.sh
   3.300 +
   3.301 +    >   /var/local/build/firethorn /
   3.302 +    >   Merging [2.1.31-stv-cleanup] into [2.1.32] (Y/n)y
   3.303 +    >   Close dev branch [2.1.31-stv-cleanup] (Y/n)y
   3.304 +    >   51 files updated, 0 files merged, 0 files removed, 0 files unresolved
   3.305 +    >   51 files updated, 0 files merged, 0 files removed, 0 files unresolved
   3.306 +    >   /
   3.307 +
   3.308 +
   3.309 +    # -----------------------------------------------------
   3.310 +    # Push changes to Mercurial
   3.311 +    #[root@builder]
   3.312 +
   3.313 +        08.03-mercurial-push.sh
   3.314 +
   3.315 +    >   /var/local/build/firethorn /
   3.316 +    >   Push version [2.1.32] (Y/n)
   3.317 +    >   pushing to ssh://Zarquan@data.metagrid.co.uk//var/local/wfau/projects/firethorn/code
   3.318 +    >   The authenticity of host 'data.metagrid.co.uk (178.79.157.93)' can't be established.
   3.319 +    >   RSA key fingerprint is SHA256:iTDlBuOqyiDYLFqdkLKZH4TWz7MlavLvkP2u3yFFVwo.
   3.320 +    >   Are you sure you want to continue connecting (yes/no)? yes
   3.321 +    >   remote: Warning: Permanently added 'data.metagrid.co.uk,178.79.157.93' (RSA) to the list of known hosts.
   3.322 +    >   searching for changes
   3.323 +    >   remote: adding changesets
   3.324 +    >   remote: adding manifests
   3.325 +    >   remote: adding file changes
   3.326 +    >   remote: added 3 changesets with 52 changes to 52 files (+1 heads)
   3.327 +    >   /
   3.328 +
   3.329 +
   3.330 +    # -----------------------------------------------------
   3.331 +    # Tag our new images
   3.332 +    #[root@builder]
   3.333 +
   3.334 +        08.01-docker-latest.sh
   3.335 +
   3.336 +
   3.337 +
   3.338 +    # -----------------------------------------------------
   3.339 +    # Push images to Docker.
   3.340 +    #[root@builder]
   3.341 +
   3.342 +        08.02-docker-push.sh
   3.343 +
   3.344 +    >   WARNING! Using --password via the CLI is insecure. Use --password-stdin.
   3.345 +    >   WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
   3.346 +    >   Configure a credential helper to remove this warning. See
   3.347 +    >   https://docs.docker.com/engine/reference/commandline/login/#credentials-store
   3.348 +    >   
   3.349 +    >   Login Succeeded
   3.350 +    >   The push refers to repository [docker.io/firethorn/fedora]
   3.351 +    >   8bd87b3d77b4: Pushed
   3.352 +    >   fc5a9c11c29d: Pushed
   3.353 +    >   ....
   3.354 +    >   ....
   3.355 +    >   891e1e4ef82a: Layer already exists
   3.356 +    >   2.1.32: digest: sha256:fb6357736d42559f8363223d9fe071749e3a119246da79825a08566fe84eb7ea size: 1791
   3.357 +    >   The push refers to repository [docker.io/firethorn/java]
   3.358 +    >   443d62749c73: Pushed
   3.359 +    >   8bd87b3d77b4: Mounted from firethorn/fedora
   3.360 +    >   ....
   3.361 +    >   ....
   3.362 +    >   2.1.32: digest: sha256:6c309b9af82a7152050f47a41ea7ee9e2f442fe1bf1852fedcd46039bd62bdb9 size: 2004
   3.363 +    >   The push refers to repository [docker.io/firethorn/tomcat]
   3.364 +    >   8e535c0bd17d: Pushed
   3.365 +    >   3cba4c50c98b: Pushed
   3.366 +    >   ....
   3.367 +    >   ....
   3.368 +    >   2.1.32: digest: sha256:1c3c6dcf8b9b7e5d2319f8bf7db3445e62a84224e4a37a7d30444303999e03ed size: 5754
   3.369 +    >   The push refers to repository [docker.io/firethorn/firethorn]
   3.370 +    >   a4ef9f2ca5dc: Pushed
   3.371 +    >   8bda3c0737f5: Pushed
   3.372 +    >   ....
   3.373 +    >   ....
   3.374 +    >   2.1.32: digest: sha256:d7bcce38b5170d7253adb664e34795d35e644741cfab107410228abe813532c2 size: 6174
   3.375 +    >   The push refers to repository [docker.io/firethorn/ogsadai]
   3.376 +    >   1eb420ad7c0b: Pushed
   3.377 +    >   369276ebc0e2: Pushed
   3.378 +    >   ....
   3.379 +    >   ....
   3.380 +    >   2.1.32: digest: sha256:5c090256a78574a1f75e21c245f34d8f9fb965bf1f740644188e36ba41bcd4cc size: 7210
   3.381 +    >   The push refers to repository [docker.io/firethorn/builder]
   3.382 +    >   be399e64df43: Pushed
   3.383 +    >   1daf40c55403: Pushed
   3.384 +    >   ....
   3.385 +    >   ....
   3.386 +    >   2.1.32: digest: sha256:48e0ccd96945c6a9ffc747e6849112b4fa535b1e9c3934d21daf23e75a309950 size: 6617
   3.387 +    >   The push refers to repository [docker.io/firethorn/tester]
   3.388 +    >   cd575670e127: Pushed
   3.389 +    >   2cb6a81a359e: Pushed
   3.390 +    >   ....
   3.391 +    >   ....
   3.392 +    >   2.1.32: digest: sha256:4765bb92d7eb36406b82ae590b764811de87b2c282c9e8d3c8bcf56e55a248f7 size: 2631
   3.393 +    >   The push refers to repository [docker.io/firethorn/postgres]
   3.394 +    >   9a56de1e814d: Pushed
   3.395 +    >   44f595c1dcd2: Layer already exists
   3.396 +    >   ....
   3.397 +    >   ....
   3.398 +    >   2.1.32: digest: sha256:8efb9465b6dd7a22b62da18fb3d97d73926a1ffdec4c3e9448dd39e5cfbab5c1 size: 3453
   3.399 +    >   The push refers to repository [docker.io/firethorn/sql-proxy]
   3.400 +    >   635548d2e88f: Pushed
   3.401 +    >   696c2bfb61d3: Pushed
   3.402 +    >   ....
   3.403 +    >   ....
   3.404 +    >   2.1.32: digest: sha256:4eedbc6d37e561bffcfa26f5bac99085ddd93522c8dbdc949ec8b263540c07e2 size: 3051
   3.405 +    >   The push refers to repository [docker.io/firethorn/firethorn-py]
   3.406 +    >   aaedb3375471: Pushed
   3.407 +    >   4967f15b8200: Pushed
   3.408 +    >   ....
   3.409 +    >   ....
   3.410 +    >   2.1.32: digest: sha256:4ce4b8a95c42b3ea6bfcf1a835b0c61ecc2b1a08e4b46ba80222763f85ae375f size: 3472
   3.411 +    >   The push refers to repository [docker.io/firethorn/fedora]
   3.412 +    >   8bd87b3d77b4: Layer already exists
   3.413 +    >   fc5a9c11c29d: Layer already exists
   3.414 +    >   ....
   3.415 +    >   ....
   3.416 +    >   latest: digest: sha256:fb6357736d42559f8363223d9fe071749e3a119246da79825a08566fe84eb7ea size: 1791
   3.417 +    >   The push refers to repository [docker.io/firethorn/java]
   3.418 +    >   443d62749c73: Layer already exists
   3.419 +    >   8bd87b3d77b4: Layer already exists
   3.420 +    >   ....
   3.421 +    >   ....
   3.422 +    >   latest: digest: sha256:6c309b9af82a7152050f47a41ea7ee9e2f442fe1bf1852fedcd46039bd62bdb9 size: 2004
   3.423 +    >   The push refers to repository [docker.io/firethorn/tomcat]
   3.424 +    >   8e535c0bd17d: Layer already exists
   3.425 +    >   3cba4c50c98b: Layer already exists
   3.426 +    >   ....
   3.427 +    >   ....
   3.428 +    >   latest: digest: sha256:1c3c6dcf8b9b7e5d2319f8bf7db3445e62a84224e4a37a7d30444303999e03ed size: 5754
   3.429 +    >   The push refers to repository [docker.io/firethorn/firethorn]
   3.430 +    >   a4ef9f2ca5dc: Layer already exists
   3.431 +    >   8bda3c0737f5: Layer already exists
   3.432 +    >   ....
   3.433 +    >   ....
   3.434 +    >   latest: digest: sha256:d7bcce38b5170d7253adb664e34795d35e644741cfab107410228abe813532c2 size: 6174
   3.435 +    >   The push refers to repository [docker.io/firethorn/ogsadai]
   3.436 +    >   1eb420ad7c0b: Layer already exists
   3.437 +    >   369276ebc0e2: Layer already exists
   3.438 +    >   ....
   3.439 +    >   ....
   3.440 +    >   latest: digest: sha256:5c090256a78574a1f75e21c245f34d8f9fb965bf1f740644188e36ba41bcd4cc size: 7210
   3.441 +    >   The push refers to repository [docker.io/firethorn/builder]
   3.442 +    >   be399e64df43: Layer already exists
   3.443 +    >   1daf40c55403: Layer already exists
   3.444 +    >   ....
   3.445 +    >   ....
   3.446 +    >   latest: digest: sha256:48e0ccd96945c6a9ffc747e6849112b4fa535b1e9c3934d21daf23e75a309950 size: 6617
   3.447 +    >   The push refers to repository [docker.io/firethorn/tester]
   3.448 +    >   cd575670e127: Layer already exists
   3.449 +    >   2cb6a81a359e: Layer already exists
   3.450 +    >   ....
   3.451 +    >   ....
   3.452 +    >   latest: digest: sha256:4765bb92d7eb36406b82ae590b764811de87b2c282c9e8d3c8bcf56e55a248f7 size: 2631
   3.453 +    >   The push refers to repository [docker.io/firethorn/postgres]
   3.454 +    >   9a56de1e814d: Layer already exists
   3.455 +    >   44f595c1dcd2: Layer already exists
   3.456 +    >   ....
   3.457 +    >   ....
   3.458 +    >   latest: digest: sha256:8efb9465b6dd7a22b62da18fb3d97d73926a1ffdec4c3e9448dd39e5cfbab5c1 size: 3453
   3.459 +    >   The push refers to repository [docker.io/firethorn/sql-proxy]
   3.460 +    >   635548d2e88f: Layer already exists
   3.461 +    >   696c2bfb61d3: Layer already exists
   3.462 +    >   ....
   3.463 +    >   ....
   3.464 +    >   latest: digest: sha256:4eedbc6d37e561bffcfa26f5bac99085ddd93522c8dbdc949ec8b263540c07e2 size: 3051
   3.465 +    >   The push refers to repository [docker.io/firethorn/firethorn-py]
   3.466 +    >   aaedb3375471: Layer already exists
   3.467 +    >   4967f15b8200: Layer already exists
   3.468 +    >   ....
   3.469 +    >   ....
   3.470 +    >   latest: digest: sha256:4ce4b8a95c42b3ea6bfcf1a835b0c61ecc2b1a08e4b46ba80222763f85ae375f size: 3472
   3.471 +
   3.472 +
   3.473 +    # -----------------------------------------------------
   3.474 +    # Push binaries to Maven repo.
   3.475 +    #[root@builder]
   3.476 +
   3.477 +        08.04-maven-push.sh
   3.478 +
   3.479 +    >   building file list ...
   3.480 +    >   347 files to consider
   3.481 +    >   uk/ac/roe/wfau/firethorn-admin-webapp/maven-metadata-local.xml
   3.482 +    >               318 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=340/347)
   3.483 +    >   uk/ac/roe/wfau/firethorn-admin-webapp/maven-metadata-local.xml.md5
   3.484 +    >                32 100%   31.25kB/s    0:00:00 (xfr#2, to-chk=339/347)
   3.485 +    >   ....
   3.486 +    >   ....
   3.487 +    >   Number of files: 347 (reg: 278, dir: 69)
   3.488 +    >   Number of created files: 214 (reg: 182, dir: 32)
   3.489 +    >   Number of regular files transferred: 278
   3.490 +    >   Total file size: 130.54M bytes
   3.491 +    >   Total transferred file size: 130.54M bytes
   3.492 +    >   Literal data: 130.54M bytes
   3.493 +    >   Matched data: 0 bytes
   3.494 +    >   File list size: 0
   3.495 +    >   File list generation time: 0.446 seconds
   3.496 +    >   File list transfer time: 0.000 seconds
   3.497 +    >   Total bytes sent: 130.60M
   3.498 +    >   Total bytes received: 5.97K
   3.499 +    >   
   3.500 +    >   sent 130.60M bytes  received 5.97K bytes  13.75M bytes/sec
   3.501 +    >   total size is 130.54M  speedup is 1.00
   3.502 +    >   
   3.503 +    >   building file list ...
   3.504 +    >   95 files to consider
   3.505 +    >   uk/org/ogsadai/ogsadai-basic-client/4.2.2198/_remote.repositories
   3.506 +    >               246 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=88/95)
   3.507 +    >   uk/org/ogsadai/ogsadai-basic-server/4.2.2198/_remote.repositories
   3.508 +    >               246 100%  240.23kB/s    0:00:00 (xfr#2, to-chk=81/95)
   3.509 +    >   ....
   3.510 +    >   ....
   3.511 +    >   Number of files: 95 (reg: 65, dir: 30)
   3.512 +    >   Number of created files: 0
   3.513 +    >   Number of regular files transferred: 13
   3.514 +    >   Total file size: 29.41M bytes
   3.515 +    >   Total transferred file size: 3.19K bytes
   3.516 +    >   Literal data: 3.19K bytes
   3.517 +    >   Matched data: 0 bytes
   3.518 +    >   File list size: 0
   3.519 +    >   File list generation time: 0.159 seconds
   3.520 +    >   File list transfer time: 0.000 seconds
   3.521 +    >   Total bytes sent: 7.33K
   3.522 +    >   Total bytes received: 336
   3.523 +    >   
   3.524 +    >   sent 7.33K bytes  received 336 bytes  3.07K bytes/sec
   3.525 +    >   total size is 29.41M  speedup is 3,836.53
   3.526 +    >   
   3.527 +    >   building file list ...
   3.528 +    >   31 files to consider
   3.529 +    >   uk/ac/starlink/stil-table/3.3.1/_remote.repositories
   3.530 +    >               214 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=24/31)
   3.531 +    >   uk/ac/starlink/stil-table/3.3.1/stil-table-3.3.1.jar.lastUpdated
   3.532 +    >               348 100%  339.84kB/s    0:00:00 (xfr#2, to-chk=22/31)
   3.533 +    >   ....
   3.534 +    >   ....
   3.535 +    >   Number of files: 31 (reg: 21, dir: 10)
   3.536 +    >   Number of created files: 0
   3.537 +    >   Number of regular files transferred: 9
   3.538 +    >   Total file size: 1.23M bytes
   3.539 +    >   Total transferred file size: 2.73K bytes
   3.540 +    >   Literal data: 2.73K bytes
   3.541 +    >   Matched data: 0 bytes
   3.542 +    >   File list size: 0
   3.543 +    >   File list generation time: 0.012 seconds
   3.544 +    >   File list transfer time: 0.000 seconds
   3.545 +    >   Total bytes sent: 4.14K
   3.546 +    >   Total bytes received: 236
   3.547 +    >   
   3.548 +    >   sent 4.14K bytes  received 236 bytes  8.75K bytes/sec
   3.549 +    >   total size is 1.23M  speedup is 281.07
   3.550 +    >   
   3.551 +    >   building file list ...
   3.552 +    >   3868 files to consider
   3.553 +    >   antlr/antlr/2.7.7/_remote.repositories
   3.554 +    >               204 100%    0.00kB/s    0:00:00 (xfr#1, to-chk=3863/3868)
   3.555 +    >   antlr/antlr/2.7.7/antlr-2.7.7.jar.lastUpdated
   3.556 +    >               348 100%  339.84kB/s    0:00:00 (xfr#2, to-chk=3861/3868)
   3.557 +    >   ....
   3.558 +    >   ....
   3.559 +    >   ....
   3.560 +    >   ....
   3.561 +    >   Number of files: 3,868 (reg: 2,744, dir: 1,124)
   3.562 +    >   Number of created files: 0
   3.563 +    >   Number of regular files transferred: 880
   3.564 +    >   Total file size: 107.40M bytes
   3.565 +    >   Total transferred file size: 229.83K bytes
   3.566 +    >   Literal data: 229.78K bytes
   3.567 +    >   Matched data: 45 bytes
   3.568 +    >   File list size: 131.06K
   3.569 +    >   File list generation time: 1.002 seconds
   3.570 +    >   File list transfer time: 0.000 seconds
   3.571 +    >   Total bytes sent: 401.16K
   3.572 +    >   Total bytes received: 22.37K
   3.573 +    >   
   3.574 +    >   sent 401.16K bytes  received 22.37K bytes  19.70K bytes/sec
   3.575 +    >   total size is 107.40M  speedup is 253.57
   3.576 +
   3.577 +
   3.578 +
   3.579 +
   3.580 +
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/doc/notes/zrq/20190601-02-merge-tests.txt	Mon Jun 03 04:46:00 2019 +0100
     4.3 @@ -0,0 +1,427 @@
     4.4 +#
     4.5 +# <meta:header>
     4.6 +#   <meta:licence>
     4.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     4.8 +#
     4.9 +#     This information is free software: you can redistribute it and/or modify
    4.10 +#     it under the terms of the GNU General Public License as published by
    4.11 +#     the Free Software Foundation, either version 3 of the License, or
    4.12 +#     (at your option) any later version.
    4.13 +#
    4.14 +#     This information is distributed in the hope that it will be useful,
    4.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    4.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    4.17 +#     GNU General Public License for more details.
    4.18 +#
    4.19 +#     You should have received a copy of the GNU General Public License
    4.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    4.21 +#   </meta:licence>
    4.22 +# </meta:header>
    4.23 +#
    4.24 +#
    4.25 +
    4.26 +# -----------------------------------------------------
    4.27 +# Create our chain properties.
    4.28 +#[user@builder]
    4.29 +
    4.30 +    cat > "${FIRETHORN_HOME:?}/chain.properties" << EOF
    4.31 +
    4.32 +        buildtag=${buildtag:?}
    4.33 +
    4.34 +        metadata=$(pwgen 20 1)
    4.35 +        metauser=$(pwgen 20 1)
    4.36 +        metapass=$(pwgen 20 1)
    4.37 +
    4.38 +        usertype=mssql
    4.39 +        userhost=$(secret 'firethorn.user.host')
    4.40 +        userdata=$(secret 'firethorn.user.data')
    4.41 +        useruser=$(secret 'firethorn.user.user')
    4.42 +        userpass=$(secret 'firethorn.user.pass')
    4.43 +
    4.44 +        datatype=mssql
    4.45 +        datahost=$(secret 'firethorn.data.host')
    4.46 +        datadata=$(secret 'firethorn.data.data')
    4.47 +        datauser=$(secret 'firethorn.data.user')
    4.48 +        datapass=$(secret 'firethorn.data.pass')
    4.49 +
    4.50 +        tunneluser=$(secret 'ssh.tunnel.user')
    4.51 +        tunnelhost=$(secret 'ssh.tunnel.host')
    4.52 +
    4.53 +        admingroup=Hyaenidae
    4.54 +        adminuser=Aardwolf
    4.55 +        adminpass=$(pwgen 20 1)
    4.56 +
    4.57 +        guestgroup=Afrotheria
    4.58 +        guestuser=Hyrax
    4.59 +        guestpass=$(pwgen 20 1)
    4.60 +
    4.61 +        tapresource=Wilhelmina
    4.62 +        tapschemadata=data-$(pwgen 10 1)
    4.63 +        tapschemauser=user-$(pwgen 10 1)
    4.64 +        tapschemapass=pass-$(pwgen 10 1)
    4.65 +
    4.66 +EOF
    4.67 +
    4.68 +# -----------------------------------------------------
    4.69 +# Link our compose config.
    4.70 +#[user@builder]
    4.71 +
    4.72 +    ln -sf "${FIRETHORN_HOME:?}/chain.properties" "${FIRETHORN_HOME:?}/.env"
    4.73 +
    4.74 +# -----------------------------------------------------
    4.75 +# Start our test containers ...
    4.76 +#[user@builder]
    4.77 +
    4.78 +    pushd "${FIRETHORN_HOME:?}"
    4.79 +
    4.80 +#
    4.81 +# Choose one.
    4.82 +# 'local' if VM is inside UoE
    4.83 +# 'remote' if VM is at outside UoE
    4.84 +# TODO Change these to 'internal' and 'external'
    4.85 +
    4.86 +        external=$(curl -4 --silent 'http://icanhazip.com/')
    4.87 +
    4.88 +        EDINBURGH='129.215.*'
    4.89 +
    4.90 +        if [[ ${external:?} == ${EDINBURGH} ]]
    4.91 +        then
    4.92 +            echo "Address [${external:?}] is in Edinburgh"
    4.93 +            #location=internal
    4.94 +            location=local
    4.95 +        else
    4.96 +            echo "Address [${external:?}] is not in Edinburgh"
    4.97 +            #location=external
    4.98 +            location=remote
    4.99 +        fi
   4.100 +
   4.101 +        # Check the env file is linked !!
   4.102 +
   4.103 +        docker-compose \
   4.104 +            --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-${location:?}.yml" \
   4.105 +            run \
   4.106 +                angela
   4.107 +
   4.108 +# -----------------------------------------------------
   4.109 +# -----------------------------------------------------
   4.110 +# Separate shell on the host VM, locate the logs volume and tail the firethorn log.
   4.111 +#[user@virtual]
   4.112 +
   4.113 +    sudo -s
   4.114 +
   4.115 +    container=baryptera_gillian_1
   4.116 +
   4.117 +    pushd $(
   4.118 +        docker inspect \
   4.119 +            "${container:?}" \
   4.120 +      | jq -r '
   4.121 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   4.122 +            '
   4.123 +            )
   4.124 +
   4.125 +    tail -f firethorn-debug.log
   4.126 +
   4.127 +# -----------------------------------------------------
   4.128 +# -----------------------------------------------------
   4.129 +# Separate shell on the host VM, locate the logs volume and tail the ogsadai log.
   4.130 +#[user@virtual]
   4.131 +
   4.132 +    sudo -s
   4.133 +
   4.134 +    container=baryptera_jarmila_1
   4.135 +
   4.136 +    pushd $(
   4.137 +        docker inspect \
   4.138 +            "${container:?}" \
   4.139 +      | jq -r '
   4.140 +            .[].Mounts | .[] | select(.Destination == "/var/local/tomcat/logs") | .Source
   4.141 +            '
   4.142 +            )
   4.143 +
   4.144 +    tail -f ogsadai.log
   4.145 +
   4.146 +# -----------------------------------------------------
   4.147 +# -----------------------------------------------------
   4.148 +# Run our Python tests ...
   4.149 +#[user@python]
   4.150 +
   4.151 +import os
   4.152 +import uuid
   4.153 +import time
   4.154 +import firethorn as ftpy
   4.155 +
   4.156 +#
   4.157 +# Create our firethorn client (using named param).
   4.158 +firethorn = ftpy.Firethorn(
   4.159 +    endpoint = os.environ.get(
   4.160 +        'endpoint'
   4.161 +        )
   4.162 +    )
   4.163 +
   4.164 +#
   4.165 +# Login as the admin account.
   4.166 +firethorn.login(
   4.167 +    os.environ.get('adminuser'),
   4.168 +    os.environ.get('adminpass'),
   4.169 +    os.environ.get('admingroup')
   4.170 +    )
   4.171 +
   4.172 +#
   4.173 +# Create a JdbcResource to connect to the ATLAS database.
   4.174 +atlas_jdbc = firethorn.firethorn_engine.create_jdbc_resource(
   4.175 +    "ATLAS JDBC resource",
   4.176 +    os.environ.get('datadata'),
   4.177 +    '*',
   4.178 +    os.environ.get('datatype'),
   4.179 +    os.environ.get('datahost'),
   4.180 +    os.environ.get('datauser'),
   4.181 +    os.environ.get('datapass')
   4.182 +    )
   4.183 +print(
   4.184 +    atlas_jdbc
   4.185 +    )
   4.186 +
   4.187 +#
   4.188 +# Create an AdqlResource to represent the JdbcResource.
   4.189 +atlas_adql = firethorn.firethorn_engine.create_adql_resource(
   4.190 +    "ATLAS ADQL resource"
   4.191 +    )
   4.192 +print(
   4.193 +    atlas_adql
   4.194 +    )
   4.195 +
   4.196 +#
   4.197 +# Import the target JdbcSchema into AdqlSchema.
   4.198 +schema_names = [
   4.199 +    "ATLASDR1"
   4.200 +    ]
   4.201 +
   4.202 +for schema_name in schema_names:
   4.203 +    print(schema_name)
   4.204 +    jdbc_schema = atlas_jdbc.select_schema_by_name(
   4.205 +        schema_name,
   4.206 +        "dbo"
   4.207 +        )
   4.208 +    if (None != jdbc_schema):
   4.209 +        metadoc="https://raw.githubusercontent.com/wfau/metadata/master/metadocs/" + schema_name + "_TablesSchema.xml"
   4.210 +        adql_schema = atlas_adql.import_jdbc_schema(
   4.211 +            jdbc_schema,
   4.212 +            schema_name,
   4.213 +            metadoc=metadoc
   4.214 +            )
   4.215 +
   4.216 +#
   4.217 +# Admin user
   4.218 +# -------- -------- -------- --------
   4.219 +# Normal user
   4.220 +#
   4.221 +
   4.222 +#
   4.223 +# Login using a guest account.
   4.224 +firethorn.login(
   4.225 +    str(uuid.uuid4()),
   4.226 +    str(uuid.uuid4()),
   4.227 +    None
   4.228 +    )
   4.229 +
   4.230 +#
   4.231 +# Create a new workspace.
   4.232 +workspace = firethorn.firethorn_engine.create_adql_resource(
   4.233 +    "Query resource"
   4.234 +    )
   4.235 +
   4.236 +#
   4.237 +# Import the ATLAS schemas into our workspace
   4.238 +for schema in atlas_adql.select_schemas():
   4.239 +    workspace.import_adql_schema(
   4.240 +        schema
   4.241 +        )
   4.242 +
   4.243 +#
   4.244 +# Create and run a query.
   4.245 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   4.246 +query_obj = workspace.create_query(
   4.247 +    query_str,
   4.248 +    "COMPLETED",
   4.249 +    None,
   4.250 +    3000000
   4.251 +    )
   4.252 +print(
   4.253 +    query_obj
   4.254 +    )
   4.255 +print(
   4.256 +    query_obj.table()
   4.257 +    )
   4.258 +print(
   4.259 +    query_obj.table().count()
   4.260 +    )
   4.261 +
   4.262 +#
   4.263 +# Iterate the metadata tree
   4.264 +for schema in atlas_adql.select_schemas():
   4.265 +    for table in schema.select_tables():
   4.266 +        print(
   4.267 +            "table  [{}][{}]".format(
   4.268 +                schema.name(),
   4.269 +                table.name()
   4.270 +                )
   4.271 +            )
   4.272 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
   4.273 +            schema.name(),
   4.274 +            table.name()
   4.275 +            )
   4.276 +        query_obj = workspace.create_query(
   4.277 +            query_str,
   4.278 +            "COMPLETED",
   4.279 +            None,
   4.280 +            3000000
   4.281 +            )
   4.282 +        py_table = query_obj.table().as_astropy()
   4.283 +        py_table.pprint()
   4.284 +
   4.285 +#
   4.286 +# Run some queries in parallel
   4.287 +from concurrent.futures import ThreadPoolExecutor
   4.288 +import concurrent.futures
   4.289 +from datetime import datetime
   4.290 +
   4.291 +query_str = "SELECT TOP 10000 ra, dec FROM ATLASDR1.atlasSource"
   4.292 +
   4.293 +def do_query(workspace, query_str, limit, delay):
   4.294 +    before = datetime.now()
   4.295 +    query_obj = workspace.create_query(
   4.296 +        query_str,
   4.297 +        "COMPLETED",
   4.298 +        None,
   4.299 +        200000,
   4.300 +            {
   4.301 +            "adql.query.limit.rows"  : limit,
   4.302 +            "adql.query.delay.every" : delay
   4.303 +            }
   4.304 +        )
   4.305 +    after = datetime.now()
   4.306 +    return (
   4.307 +        (after - before),
   4.308 +        query_obj.json_object.get("results").get("count")
   4.309 +        )
   4.310 +
   4.311 +def do_queries(workspace, query_str, threads, delay):
   4.312 +    with concurrent.futures.ThreadPoolExecutor(threads) as executor:
   4.313 +        futures = {
   4.314 +            executor.submit(
   4.315 +                do_query,
   4.316 +                workspace,
   4.317 +                query_str,
   4.318 +                limit,
   4.319 +                delay
   4.320 +                ): limit for limit in range(threads, 0, -1)
   4.321 +            }
   4.322 +        for future in concurrent.futures.as_completed(futures):
   4.323 +            print(
   4.324 +                future.result()[0],
   4.325 +                ':',
   4.326 +                future.result()[1]
   4.327 +                )
   4.328 +
   4.329 +for loop in range(1, 10):
   4.330 +    for threads in range(1, 50):
   4.331 +        for delay in range(1000, -100, -100):
   4.332 +            print("---- ", loop, threads, delay)
   4.333 +            do_queries(
   4.334 +                workspace,
   4.335 +                query_str,
   4.336 +                threads,
   4.337 +                delay
   4.338 +                )
   4.339 +
   4.340 +
   4.341 +
   4.342 +# -------- -------- -------- --------
   4.343 +# Bug testing.
   4.344 +
   4.345 +
   4.346 +#
   4.347 +# Triggering a ChaosMonkey Exception in the cleanUp() method doesn't cause an error (the query completes).
   4.348 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l469
   4.349 +params = {}
   4.350 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity"})
   4.351 +params.update({"firethorn.monkey.data" : "baivahP0"})
   4.352 +
   4.353 +print(
   4.354 +    params
   4.355 +    )
   4.356 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   4.357 +query_obj = workspace.create_query(
   4.358 +    query_str,
   4.359 +    "COMPLETED",
   4.360 +    None,
   4.361 +    3000000,
   4.362 +    params=params
   4.363 +    )
   4.364 +print(
   4.365 +    query_obj
   4.366 +    )
   4.367 +
   4.368 +#
   4.369 +# Triggering a ChaosMonkey Exception between creating the Future and waiting for it doesn't cause the same problem (the query fails immediately).
   4.370 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l369
   4.371 +params = {}
   4.372 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity"})
   4.373 +params.update({"firethorn.monkey.data" : "uche2aNa"})
   4.374 +
   4.375 +print(
   4.376 +    params
   4.377 +    )
   4.378 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   4.379 +query_obj = workspace.create_query(
   4.380 +    query_str,
   4.381 +    "COMPLETED",
   4.382 +    None,
   4.383 +    3000000,
   4.384 +    params=params
   4.385 +    )
   4.386 +print(
   4.387 +    query_obj
   4.388 +    )
   4.389 +
   4.390 +#
   4.391 +# Trigger a ChaosMonkey Exception inside the CallableStatement replicates the same symptoms as the issue we are tracking.
   4.392 +# http://wfau.metagrid.co.uk/code/firethorn/file/ab423c5da4f5/firethorn-ogsadai/activity/server/src/main/java/uk/ac/roe/wfau/firethorn/ogsadai/activity/server/sql/SQLQueryActivity.java#l414
   4.393 +params = {}
   4.394 +params.update({"firethorn.monkey.name" : "uk.ac.roe.wfau.firethorn.ogsadai.activity.server.sql.SQLQueryActivity$ChaoticCallableStatement"})
   4.395 +params.update({"firethorn.monkey.data" : "Eoph9xie"})
   4.396 +
   4.397 +print(
   4.398 +    params
   4.399 +    )
   4.400 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   4.401 +query_obj = workspace.create_query(
   4.402 +    query_str,
   4.403 +    "COMPLETED",
   4.404 +    None,
   4.405 +    3000000,
   4.406 +    params=params
   4.407 +    )
   4.408 +print(
   4.409 +    query_obj
   4.410 +    )
   4.411 +
   4.412 +
   4.413 +
   4.414 +
   4.415 +
   4.416 +
   4.417 +#
   4.418 +# Exit the Python shell
   4.419 +
   4.420 +    Ctrl^D
   4.421 +
   4.422 +# -----------------------------------------------------
   4.423 +# Shutdown our containers ...
   4.424 +#[user@virtual]
   4.425 +
   4.426 +    docker-compose \
   4.427 +        --file "${FIRETHORN_CODE:?}/docker/compose/tests/baryptera/baryptera-local.yml" \
   4.428 +        down
   4.429 +
   4.430 +