firethorn

changeset 4215:373d1a1f0078

VM down debugging notes
author Dave Morris <dmr@roe.ac.uk>
date Thu May 09 05:34:15 2019 +0100 (2019-05-09)
parents f2f3dc542d52
children 57acf41bcc1c
files doc/notes/zrq/20190508-01-vmdown-debug.txt firethorn-webapp/src/main/java/uk/ac/roe/wfau/firethorn/webapp/tap/ScriptRunner.java
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/zrq/20190508-01-vmdown-debug.txt	Thu May 09 05:34:15 2019 +0100
     1.3 @@ -0,0 +1,1445 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +# -----------------------------------------------------
    1.27 +# Check what containers are running.
    1.28 +#[user@virtual]
    1.29 +
    1.30 +    docker ps -a
    1.31 +
    1.32 +--START--
    1.33 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
    1.34 +16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                12 days ago         Exited (137) 12 days ago                              stevedore_firethorn-py_run_5
    1.35 +2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 12 days ago                              stevedore_firethorn-py_run_4
    1.36 +659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
    1.37 +cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
    1.38 +ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
    1.39 +513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
    1.40 +f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 5 minutes ago   8080/tcp                 stevedore_jarmila_1
    1.41 +7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_carolina_1
    1.42 +80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 5 minutes ago   5432/tcp                 stevedore_bethany_1
    1.43 +--END--
    1.44 +
    1.45 +
    1.46 +# -----------------------------------------------------
    1.47 +# Run the Python client - bringing the dependencies back up.
    1.48 +#[user@virtual]
    1.49 +
    1.50 +    export buildtag=2.1.28
    1.51 +
    1.52 +    source "${HOME:?}/chain.properties"
    1.53 +
    1.54 +    docker-compose --file "docker-compose.yml" run firethorn-py
    1.55 +
    1.56 +--START--
    1.57 +Starting stevedore_bethany_1  ... done
    1.58 +Starting stevedore_jarmila_1  ... done
    1.59 +Starting stevedore_carolina_1 ... done
    1.60 +Starting stevedore_gillian_1  ... done
    1.61 +Python 3.5.2 (default, Nov 12 2018, 13:43:14)
    1.62 +[GCC 5.4.0 20160609] on linux
    1.63 +Type "help", "copyright", "credits" or "license" for more information.
    1.64 +--END--
    1.65 +
    1.66 +
    1.67 +# -----------------------------------------------------
    1.68 +# Run a quick Python test.
    1.69 +#[python]
    1.70 +
    1.71 +import os
    1.72 +import uuid
    1.73 +import time
    1.74 +import firethorn as ftpy
    1.75 +
    1.76 +#
    1.77 +# Create our Firethorn client.
    1.78 +firethorn = ftpy.Firethorn(
    1.79 +    endpoint = os.environ.get(
    1.80 +        'endpoint'
    1.81 +        )
    1.82 +    )
    1.83 +
    1.84 +#
    1.85 +# Login using a guest account.
    1.86 +firethorn.login(
    1.87 +    str(uuid.uuid4()),
    1.88 +    str(uuid.uuid4()),
    1.89 +    None
    1.90 +    )
    1.91 +
    1.92 +#
    1.93 +# Get a list of AdqlResources.
    1.94 +for resource in firethorn.firethorn_engine.select_adql_resources():
    1.95 +    print(
    1.96 +        resource.name()
    1.97 +        )
    1.98 +
    1.99 +
   1.100 +    #
   1.101 +    # **lots** of user space resources.
   1.102 +    #
   1.103 +
   1.104 +    mySpace
   1.105 +    mySpace
   1.106 +    mySpace
   1.107 +    mySpace
   1.108 +    mySpace
   1.109 +    mySpace
   1.110 +    mySpace
   1.111 +    mySpace
   1.112 +    mySpace
   1.113 +    mySpace
   1.114 +    mySpace
   1.115 +    mySpace
   1.116 +    mySpace
   1.117 +    mySpace
   1.118 +    mySpace
   1.119 +    mySpace
   1.120 +    mySpace
   1.121 +    mySpace
   1.122 +    mySpace
   1.123 +    mySpace
   1.124 +    mySpace
   1.125 +    mySpace
   1.126 +    mySpace
   1.127 +    mySpace
   1.128 +    mySpace
   1.129 +    mySpace
   1.130 +    mySpace
   1.131 +    mySpace
   1.132 +    OSA ADQL resource
   1.133 +    OSA ADQL resource
   1.134 +    SSA ADQL resource
   1.135 +    user space
   1.136 +    ....
   1.137 +    ....
   1.138 +    user space
   1.139 +    user space
   1.140 +    user space
   1.141 +    user space
   1.142 +    VSA ADQL resource
   1.143 +    WSA ADQL resource
   1.144 +
   1.145 +#
   1.146 +# Get the Atlas AdqlResource.
   1.147 +resource = firethorn.firethorn_engine.select_adql_resources()[28]
   1.148 +
   1.149 +#
   1.150 +# Create and run a query on the resource.
   1.151 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   1.152 +query_obj = resource.create_query(
   1.153 +    query_str,
   1.154 +    "COMPLETED",
   1.155 +    None,
   1.156 +    3000000
   1.157 +    )
   1.158 +print(
   1.159 +    query_obj
   1.160 +    )
   1.161 +print(
   1.162 +    query_obj.table()
   1.163 +    )
   1.164 +print(
   1.165 +    query_obj.table().count()
   1.166 +    )
   1.167 +
   1.168 +
   1.169 +#
   1.170 +# *lots* of broken :-(
   1.171 +#
   1.172 +
   1.173 +--START--
   1.174 +{
   1.175 +  "resources": [],
   1.176 +  "owner": "http://gillian:8080/firethorn/community-member/57698559",
   1.177 +  "text": null,
   1.178 +  "callback": "http://gillian:8081/firethorn/callback/57698652",
   1.179 +  "modified": "2019-05-08T19:34:59.347",
   1.180 +  "history": [],
   1.181 +  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
   1.182 +  "delays": {
   1.183 +    "first": null,
   1.184 +    "last": null,
   1.185 +    "every": null
   1.186 +  },
   1.187 +  "url": "http://gillian:8080/firethorn/blue/query/57698652",
   1.188 +  "ident": "57698652",
   1.189 +  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
   1.190 +  "columns": [],
   1.191 +  "fields": [],
   1.192 +  "tables": [],
   1.193 +  "mode": "AUTO",
   1.194 +  "created": "2019-05-08T19:34:59.210",
   1.195 +  "name": "XX_5U5BMFLXCC4E4AAAAFVJR4I3RI",
   1.196 +  "results": {
   1.197 +    "count": 0,
   1.198 +    "state": "NONE",
   1.199 +    "formats": {
   1.200 +      "datatable": null,
   1.201 +      "votable": null
   1.202 +    },
   1.203 +    "table": null
   1.204 +  },
   1.205 +  "osql": null,
   1.206 +  "adql": null,
   1.207 +  "workspace": "http://gillian:8080/firethorn/adql/resource/57688319",
   1.208 +  "limits": {
   1.209 +    "cells": null,
   1.210 +    "rows": 1000000,
   1.211 +    "time": null
   1.212 +  },
   1.213 +  "resource": "http://gillian:8080/firethorn/adql/resource/57688319",
   1.214 +  "status": "FAILED",
   1.215 +  "syntax": {
   1.216 +    "status": "PARSE_ERROR",
   1.217 +    "friendly": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !",
   1.218 +    "message": "3 unresolved identifiers: atlasSource [l.1 c.30 - l.1 c.50], ra [l.1 c.17 - l.1 c.19], dec [l.1 c.21 - l.1 c.24]!\n  - Unknown table \"ATLASDR1.atlasSource\" !\n  - Unknown column \"ra\" !\n  - Unknown column \"dec\" !"
   1.219 +  },
   1.220 +  "self": "http://gillian:8080/firethorn/blue/query/57698652"
   1.221 +}
   1.222 +>>> print(
   1.223 +...     query_obj.table()
   1.224 +...     )
   1.225 +ERROR:root:unknown url type: 'None'
   1.226 +Traceback (most recent call last):
   1.227 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.228 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.229 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.230 +    self.full_url = url
   1.231 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.232 +    self._parse()
   1.233 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.234 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.235 +ValueError: unknown url type: 'None'
   1.236 +ERROR:root:unknown url type: 'None'
   1.237 +Traceback (most recent call last):
   1.238 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.239 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.240 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.241 +    self.full_url = url
   1.242 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.243 +    self._parse()
   1.244 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.245 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.246 +ValueError: unknown url type: 'None'
   1.247 +ERROR:root:unknown url type: 'None'
   1.248 +Traceback (most recent call last):
   1.249 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.250 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.251 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.252 +    self.full_url = url
   1.253 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.254 +    self._parse()
   1.255 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.256 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.257 +ValueError: unknown url type: 'None'
   1.258 +null
   1.259 +>>> print(
   1.260 +...     query_obj.table().count()
   1.261 +...     )
   1.262 +ERROR:root:unknown url type: 'None'
   1.263 +Traceback (most recent call last):
   1.264 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.265 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.266 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.267 +    self.full_url = url
   1.268 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.269 +    self._parse()
   1.270 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.271 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.272 +ValueError: unknown url type: 'None'
   1.273 +ERROR:root:unknown url type: 'None'
   1.274 +Traceback (most recent call last):
   1.275 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.276 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.277 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.278 +    self.full_url = url
   1.279 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.280 +    self._parse()
   1.281 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.282 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.283 +ValueError: unknown url type: 'None'
   1.284 +ERROR:root:unknown url type: 'None'
   1.285 +Traceback (most recent call last):
   1.286 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/base/base_object.py", line 88, in get_json
   1.287 +    req = urllib.request.Request( ident, headers=self.account.get_identity_as_headers())
   1.288 +  File "/usr/lib/python3.5/urllib/request.py", line 269, in __init__
   1.289 +    self.full_url = url
   1.290 +  File "/usr/lib/python3.5/urllib/request.py", line 295, in full_url
   1.291 +    self._parse()
   1.292 +  File "/usr/lib/python3.5/urllib/request.py", line 324, in _parse
   1.293 +    raise ValueError("unknown url type: %r" % self.full_url)
   1.294 +ValueError: unknown url type: 'None'
   1.295 +None
   1.296 +>>>
   1.297 +>>>
   1.298 +>>>
   1.299 +>>>
   1.300 +>>>
   1.301 +>>> resource = firethorn.firethorn_engine.select_adql_resources()[28]
   1.302 +ERROR:root:IncompleteRead(9713032 bytes read)
   1.303 +Traceback (most recent call last):
   1.304 +  File "/usr/lib/python3.5/http/client.py", line 541, in _get_chunk_left
   1.305 +    chunk_left = self._read_next_chunk_size()
   1.306 +  File "/usr/lib/python3.5/http/client.py", line 508, in _read_next_chunk_size
   1.307 +    return int(line, 16)
   1.308 +ValueError: invalid literal for int() with base 16: b''
   1.309 +
   1.310 +During handling of the above exception, another exception occurred:
   1.311 +
   1.312 +Traceback (most recent call last):
   1.313 +  File "/usr/lib/python3.5/http/client.py", line 558, in _readall_chunked
   1.314 +    chunk_left = self._get_chunk_left()
   1.315 +  File "/usr/lib/python3.5/http/client.py", line 543, in _get_chunk_left
   1.316 +    raise IncompleteRead(b'')
   1.317 +http.client.IncompleteRead: IncompleteRead(0 bytes read)
   1.318 +
   1.319 +During handling of the above exception, another exception occurred:
   1.320 +
   1.321 +Traceback (most recent call last):
   1.322 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/core/firethorn_engine.py", line 238, in select_adql_resources
   1.323 +    adqlresources =  json.loads(response.read().decode('utf-8'))
   1.324 +  File "/usr/lib/python3.5/http/client.py", line 455, in read
   1.325 +    return self._readall_chunked()
   1.326 +  File "/usr/lib/python3.5/http/client.py", line 565, in _readall_chunked
   1.327 +    raise IncompleteRead(b''.join(value))
   1.328 +http.client.IncompleteRead: IncompleteRead(9713032 bytes read)
   1.329 +Traceback (most recent call last):
   1.330 +  File "<stdin>", line 1, in <module>
   1.331 +IndexError: list index out of range
   1.332 +>>>
   1.333 +--END--
   1.334 +
   1.335 +
   1.336 +
   1.337 +# -----------------------------------------------------
   1.338 +# Login to the container and tail the logs.
   1.339 +#[user@virtual]
   1.340 +
   1.341 +    docker exec -it stevedore_gillian_1 bash
   1.342 +
   1.343 +        tail -n 100 logs/firethorn-debug.log
   1.344 +
   1.345 +--START--
   1.346 +2019-05-08 19:48:30,592 DEBUG [callback-interface-1] [AnonymousAuthenticator] Adding anonymous Authentication
   1.347 +2019-05-08 19:48:31,322 DEBUG [callback-interface-5] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.348 +2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   queryString [null]
   1.349 +2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   authType    [null]
   1.350 +2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.351 +2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug] Request headers
   1.352 +2019-05-08 19:48:34,235 DEBUG [callback-interface-5] [HttpRequestDebug]   [host][localhost:8081]
   1.353 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   authType    [null]
   1.354 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.355 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request headers
   1.356 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [host][localhost:8081]
   1.357 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.358 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug]   [accept][*/*]
   1.359 +2019-05-08 19:48:32,659 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.360 +2019-05-08 19:48:34,236 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.361 +2019-05-08 19:48:34,235 DEBUG [callback-interface-3] [HttpRequestDebug] Request parameters
   1.362 +2019-05-08 19:48:34,236 DEBUG [callback-interface-5] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.363 +2019-05-08 19:48:36,790 DEBUG [callback-interface-3] [HttpRequestDebug] ----
   1.364 +2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug]   [accept][*/*]
   1.365 +2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] Request parameters
   1.366 +2019-05-08 19:48:36,792 DEBUG [callback-interface-5] [HttpRequestDebug] ----
   1.367 +2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug] Request properties
   1.368 +2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.369 +2019-05-08 19:48:38,187 DEBUG [callback-interface-4] [HttpRequestDebug]   queryString [null]
   1.370 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   authType    [null]
   1.371 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.372 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request headers
   1.373 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [host][localhost:8081]
   1.374 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.375 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug]   [accept][*/*]
   1.376 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] Request parameters
   1.377 +2019-05-08 19:48:40,630 DEBUG [callback-interface-4] [HttpRequestDebug] ----
   1.378 +2019-05-08 19:48:43,346 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.379 +2019-05-08 19:48:44,023 DEBUG [main-interface-3] [AdqlQueryTimings] AdqlQueryTimings()
   1.380 +2019-05-08 19:48:45,865 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.381 +2019-05-08 19:49:27,372 DEBUG [main-interface-3] [JdbcSchemaEntity] services()
   1.382 +2019-05-08 19:49:27,372 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Creating emergency threads for unassigned pending tasks!
   1.383 +2019-05-08 19:51:34,886 DEBUG [callback-interface-3] [AnonymousAuthenticator] Adding anonymous Authentication
   1.384 +2019-05-08 19:56:52,233 DEBUG [callback-interface-10] [HttpRequestDebug] Request properties
   1.385 +2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   requestURL  [http://localhost:8081/firethorn/system/info]
   1.386 +2019-05-08 19:56:56,981 DEBUG [callback-interface-10] [HttpRequestDebug]   queryString [null]
   1.387 +2019-05-08 19:56:58,423 DEBUG [callback-interface-10] [HttpRequestDebug]   authType    [null]
   1.388 +2019-05-08 19:57:02,194 DEBUG [callback-interface-10] [HttpRequestDebug]   remoteAddr  [127.0.0.1]
   1.389 +2019-05-08 19:57:02,936 DEBUG [callback-interface-10] [HttpRequestDebug] Request headers
   1.390 +2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [host][localhost:8081]
   1.391 +2019-05-08 19:57:06,455 DEBUG [callback-interface-10] [HttpRequestDebug]   [user-agent][curl/7.59.0]
   1.392 +2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug]   [accept][*/*]
   1.393 +2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] Request parameters
   1.394 +2019-05-08 19:57:07,149 DEBUG [callback-interface-10] [HttpRequestDebug] ----
   1.395 +2019-05-08 19:57:10,840 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] com.mchange.v2.async.ThreadPoolAsynchronousRunner$DeadlockDetector@7e0b09b -- APPARENT DEADLOCK!!! Complete Status:
   1.396 +	Managed Threads: 3
   1.397 +	Active Threads: 3
   1.398 +	Active Tasks:
   1.399 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@481a442c (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0)
   1.400 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1)
   1.401 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@7998e972 (com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2)
   1.402 +	Pending Tasks:
   1.403 +		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@3214a229
   1.404 +		com.mchange.v2.resourcepool.BasicResourcePool$1RefurbishCheckinResourceTask@2835bb63
   1.405 +Pool thread stack traces:
   1.406 +	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#0,5,main]
   1.407 +		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.408 +		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.409 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.410 +		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.411 +	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#1,5,main]
   1.412 +		java.util.zip.ZipFile.getEntry(ZipFile.java:314)
   1.413 +		java.util.jar.JarFile.getEntry(JarFile.java:240)
   1.414 +		java.util.jar.JarFile.getJarEntry(JarFile.java:223)
   1.415 +		org.apache.catalina.webresources.AbstractSingleArchiveResourceSet.getArchiveEntry(AbstractSingleArchiveResourceSet.java:98)
   1.416 +		org.apache.catalina.webresources.AbstractArchiveResourceSet.getResource(AbstractArchiveResourceSet.java:256)
   1.417 +		org.apache.catalina.webresources.StandardRoot.getResourceInternal(StandardRoot.java:281)
   1.418 +		org.apache.catalina.webresources.Cache.getResource(Cache.java:62)
   1.419 +		org.apache.catalina.webresources.StandardRoot.getResource(StandardRoot.java:216)
   1.420 +		org.apache.catalina.webresources.StandardRoot.getClassLoaderResource(StandardRoot.java:225)
   1.421 +		org.apache.catalina.loader.WebappClassLoaderBase.findClassInternal(WebappClassLoaderBase.java:2241)
   1.422 +		org.apache.catalina.loader.WebappClassLoaderBase.findClass(WebappClassLoaderBase.java:833)
   1.423 +		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1278)
   1.424 +		org.apache.catalina.loader.WebappClassLoaderBase.loadClass(WebappClassLoaderBase.java:1138)
   1.425 +		org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:250)
   1.426 +		org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
   1.427 +		org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
   1.428 +		org.postgresql.Driver.makeConnection(Driver.java:454)
   1.429 +		org.postgresql.Driver.connect(Driver.java:256)
   1.430 +		com.mchange.v2.c3p0.DriverManagerDataSource.getConnection(DriverManagerDataSource.java:134)
   1.431 +		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:182)
   1.432 +		com.mchange.v2.c3p0.WrapperConnectionPoolDataSource.getPooledConnection(WrapperConnectionPoolDataSource.java:171)
   1.433 +		com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool$1PooledConnectionResourcePoolManager.acquireResource(C3P0PooledConnectionPool.java:137)
   1.434 +		com.mchange.v2.resourcepool.BasicResourcePool.doAcquire(BasicResourcePool.java:1014)
   1.435 +		com.mchange.v2.resourcepool.BasicResourcePool.access$800(BasicResourcePool.java:32)
   1.436 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1810)
   1.437 +		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.438 +	Thread[com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread-#2,5,main]
   1.439 +		com.mchange.v2.resourcepool.BasicResourcePool.decrementPendingAcquires(BasicResourcePool.java:419)
   1.440 +		com.mchange.v2.resourcepool.BasicResourcePool.access$900(BasicResourcePool.java:32)
   1.441 +		com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask.run(BasicResourcePool.java:1887)
   1.442 +		com.mchange.v2.async.ThreadPoolAsynchronousRunner$PoolThread.run(ThreadPoolAsynchronousRunner.java:547)
   1.443 +
   1.444 +2019-05-08 19:59:26,545 WARN  [Timer-0] [ThreadPoolAsynchronousRunner] Task com.mchange.v2.resourcepool.BasicResourcePool$AcquireTask@4a7c51dc (in deadlocked PoolThread) failed to complete in maximum time 60000ms. Trying interrupt().
   1.445 +--END--
   1.446 +
   1.447 +    tail -n 100 logs/firethorn-trace.log
   1.448 +
   1.449 +--START--
   1.450 +2019-05-08 20:17:33,686 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.451 +2019-05-08 20:17:36,580 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754910]
   1.452 +2019-05-08 20:17:36,580 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.453 +2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.454 +2019-05-08 20:17:37,363 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.455 +2019-05-08 20:17:40,212 TRACE [main-interface-2] [IdentityEntity] services()
   1.456 +2019-05-08 20:17:40,213 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.457 +2019-05-08 20:17:42,353 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.458 +2019-05-08 20:17:43,793 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754909]
   1.459 +2019-05-08 20:17:44,500 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.460 +2019-05-08 20:17:45,886 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.461 +2019-05-08 20:17:47,983 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.462 +2019-05-08 20:17:53,135 TRACE [main-interface-2] [IdentityEntity] services()
   1.463 +2019-05-08 20:17:54,679 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.464 +2019-05-08 20:17:57,601 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.465 +2019-05-08 20:18:01,181 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754908]
   1.466 +2019-05-08 20:18:01,897 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.467 +2019-05-08 20:18:03,326 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.468 +2019-05-08 20:18:04,806 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.469 +2019-05-08 20:18:09,791 TRACE [main-interface-2] [IdentityEntity] services()
   1.470 +2019-05-08 20:18:12,605 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.471 +2019-05-08 20:18:24,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.472 +2019-05-08 20:18:27,672 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754907]
   1.473 +2019-05-08 20:18:27,672 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.474 +2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.475 +2019-05-08 20:18:32,072 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.476 +2019-05-08 20:18:35,729 TRACE [main-interface-2] [IdentityEntity] services()
   1.477 +2019-05-08 20:18:38,654 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.478 +2019-05-08 20:18:40,785 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.479 +2019-05-08 20:18:44,659 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754906]
   1.480 +2019-05-08 20:18:44,659 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.481 +2019-05-08 20:18:47,137 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.482 +2019-05-08 20:18:49,387 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.483 +2019-05-08 20:18:53,563 TRACE [main-interface-2] [IdentityEntity] services()
   1.484 +2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.485 +2019-05-08 20:18:54,255 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.486 +2019-05-08 20:18:58,588 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754905]
   1.487 +2019-05-08 20:18:59,319 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.488 +2019-05-08 20:19:00,056 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.489 +2019-05-08 20:19:01,598 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.490 +2019-05-08 20:19:02,341 TRACE [main-interface-2] [IdentityEntity] services()
   1.491 +2019-05-08 20:19:05,328 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.492 +2019-05-08 20:19:06,127 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.493 +2019-05-08 20:19:10,942 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754904]
   1.494 +2019-05-08 20:19:14,163 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.495 +2019-05-08 20:19:17,369 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.496 +2019-05-08 20:19:26,151 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.497 +2019-05-08 20:19:30,374 TRACE [main-interface-2] [IdentityEntity] services()
   1.498 +2019-05-08 20:19:32,527 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.499 +2019-05-08 20:19:34,588 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.500 +2019-05-08 20:19:37,357 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754903]
   1.501 +2019-05-08 20:19:40,885 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.502 +2019-05-08 20:19:44,329 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.503 +2019-05-08 20:19:53,642 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.504 +2019-05-08 20:19:57,191 TRACE [main-interface-2] [IdentityEntity] services()
   1.505 +2019-05-08 20:20:00,019 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.506 +2019-05-08 20:20:10,334 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.507 +2019-05-08 20:20:20,901 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54754902]
   1.508 +2019-05-08 20:20:21,729 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.509 +2019-05-08 20:20:24,094 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.510 +2019-05-08 20:20:24,873 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.511 +2019-05-08 20:20:28,789 TRACE [main-interface-2] [IdentityEntity] services()
   1.512 +2019-05-08 20:20:29,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.513 +2019-05-08 20:20:39,949 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.514 +2019-05-08 20:20:47,558 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750801]
   1.515 +2019-05-08 20:20:47,558 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.516 +2019-05-08 20:20:48,242 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.517 +2019-05-08 20:20:50,360 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.518 +2019-05-08 20:20:53,869 TRACE [main-interface-2] [IdentityEntity] services()
   1.519 +2019-05-08 20:20:56,645 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.520 +2019-05-08 20:20:59,454 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.521 +2019-05-08 20:21:04,386 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750800]
   1.522 +2019-05-08 20:21:04,386 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.523 +2019-05-08 20:21:06,581 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.524 +2019-05-08 20:21:09,381 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.525 +2019-05-08 20:21:12,257 TRACE [main-interface-2] [IdentityEntity] services()
   1.526 +2019-05-08 20:21:13,696 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.527 +2019-05-08 20:21:16,517 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.528 +2019-05-08 20:21:21,786 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750799]
   1.529 +2019-05-08 20:21:21,786 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.530 +2019-05-08 20:21:23,973 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.531 +2019-05-08 20:21:26,908 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.532 +2019-05-08 20:21:31,215 TRACE [main-interface-2] [IdentityEntity] services()
   1.533 +2019-05-08 20:21:34,123 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.534 +2019-05-08 20:21:37,228 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.535 +2019-05-08 20:21:43,020 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750798]
   1.536 +2019-05-08 20:21:45,862 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.537 +2019-05-08 20:21:47,251 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.538 +2019-05-08 20:21:50,799 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.539 +2019-05-08 20:21:54,968 TRACE [main-interface-2] [IdentityEntity] services()
   1.540 +2019-05-08 20:21:59,238 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.541 +2019-05-08 20:22:02,876 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.542 +2019-05-08 20:22:19,458 TRACE [main-interface-2] [AbstractEntityBeanImpl] AbstractEntityBeanImpl [http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json][54750797]
   1.543 +2019-05-08 20:22:20,978 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.544 +2019-05-08 20:22:25,532 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.545 +2019-05-08 20:22:30,615 TRACE [main-interface-2] [AdqlResourceEntity] services()
   1.546 +2019-05-08 20:24:14,090 TRACE [main-interface-2] [AnonymousAuthenticator] afterCompletion()
   1.547 +2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] afterCompletion()
   1.548 +2019-05-08 20:24:28,569 TRACE [main-interface-2] [OperationInterceptor] Operation [57698710][http://gillian:8080/firethorn/adql/resource/select]
   1.549 +2019-05-08 20:24:28,569 DEBUG [main-interface-2] [OperationInterceptor] Exception [org.springframework.web.util.NestedServletException][Handler dispatch failed; nested exception is java.lang.OutOfMemoryError: Java heap space]
   1.550 +--END--
   1.551 +
   1.552 +# -----------------------------------------------------
   1.553 +
   1.554 +    Different errors
   1.555 +
   1.556 +        Thread deadlock connecting to PostgreSQL database ..
   1.557 +
   1.558 +        OutOfMemoryError building the list of AdqlResourceEntity(s)
   1.559 +
   1.560 +
   1.561 +# -----------------------------------------------------
   1.562 +# Disc space on the host is not the issue (66% used).
   1.563 +#[user@trop02]
   1.564 +
   1.565 +    df -h
   1.566 +
   1.567 +--START--
   1.568 +Filesystem      Size  Used Avail Use% Mounted on
   1.569 +/dev/sda2        92G   58G   30G  66% /
   1.570 +udev             10M     0   10M   0% /dev
   1.571 +tmpfs            26G  1.2G   25G   5% /run
   1.572 +tmpfs            64G     0   64G   0% /dev/shm
   1.573 +tmpfs           5.0M     0  5.0M   0% /run/lock
   1.574 +tmpfs            64G     0   64G   0% /sys/fs/cgroup
   1.575 +/dev/sda1       511M  132K  511M   1% /boot/efi
   1.576 +/dev/sda4       9.1G   22M  8.6G   1% /tmp
   1.577 +/dev/sda5        65G  5.4G   56G   9% /var
   1.578 +/dev/sda6        53G   53M   50G   1% /home
   1.579 +/dev/sdc1       3.6T   68M  3.4T   1% /data2
   1.580 +/dev/sdb1       3.6T   74M  3.4T   1% /data1
   1.581 +tmpfs            13G     0   13G   0% /run/user/1005
   1.582 +tmpfs            13G     0   13G   0% /run/user/0
   1.583 +--END--
   1.584 +
   1.585 +# -----------------------------------------------------
   1.586 +# Memory on the host is not the issue (29G cached).
   1.587 +#[user@trop02]
   1.588 +
   1.589 +    free -h
   1.590 +
   1.591 +--START--
   1.592 +             total       used       free     shared    buffers     cached
   1.593 +Mem:          126G        39G        86G       1.1G       297M        29G
   1.594 +-/+ buffers/cache:       9.3G       116G
   1.595 +Swap:         1.9G         0B       1.9G
   1.596 +--END--
   1.597 +
   1.598 +
   1.599 +# -----------------------------------------------------
   1.600 +# Disc space on the virtual is not the issue (75% used).
   1.601 +#[user@virtual]
   1.602 +
   1.603 +    df -h
   1.604 +
   1.605 +--START--
   1.606 +Filesystem      Size  Used Avail Use% Mounted on
   1.607 +devtmpfs        2.0G     0  2.0G   0% /dev
   1.608 +tmpfs           2.0G     0  2.0G   0% /dev/shm
   1.609 +tmpfs           2.0G  940K  2.0G   1% /run
   1.610 +tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.611 +/dev/vda3        31G   23G  7.7G  75% /
   1.612 +tmpfs           2.0G   25M  2.0G   2% /tmp
   1.613 +/dev/vda1       240M   89M  135M  40% /boot
   1.614 +tmpfs           395M     0  395M   0% /run/user/1001
   1.615 +--END--
   1.616 +
   1.617 +
   1.618 +# -----------------------------------------------------
   1.619 +# Memory on the virtual is not the issue (1.4G available).
   1.620 +#[user@virtual]
   1.621 +
   1.622 +    free -h
   1.623 +
   1.624 +--START--
   1.625 +              total        used        free      shared  buff/cache   available
   1.626 +Mem:           3.9G        2.1G        125M        158M        1.7G        1.4G
   1.627 +Swap:          1.0G        174M        849M
   1.628 +--END--
   1.629 +
   1.630 +# -----------------------------------------------------
   1.631 +# Disc space in the container is not the issue (75% used).
   1.632 +#[root@firethorn]
   1.633 +
   1.634 +    df -h
   1.635 +
   1.636 +--START--
   1.637 +Filesystem      Size  Used Avail Use% Mounted on
   1.638 +/dev/vda3        31G   23G  7.7G  75% /
   1.639 +tmpfs            64M     0   64M   0% /dev
   1.640 +tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.641 +tmpfs           2.0G     0  2.0G   0% /run
   1.642 +tmpfs           2.0G   32K  2.0G   1% /tmp
   1.643 +/dev/vda3        31G   23G  7.7G  75% /etc/firethorn.properties
   1.644 +shm              64M     0   64M   0% /dev/shm
   1.645 +tmpfs           2.0G     0  2.0G   0% /proc/acpi
   1.646 +tmpfs           2.0G     0  2.0G   0% /proc/scsi
   1.647 +tmpfs           2.0G     0  2.0G   0% /sys/firmware
   1.648 +--END--
   1.649 +
   1.650 +
   1.651 +# -----------------------------------------------------
   1.652 +# Check the metadata database.
   1.653 +#[user@virtual]
   1.654 +
   1.655 +    source chain.properties
   1.656 +
   1.657 +    docker exec -it stevedore_bethany_1 \
   1.658 +        psql \
   1.659 +            --username "${metauser:?}" \
   1.660 +            --dbname   "${metadata:?}"
   1.661 +
   1.662 +--START--
   1.663 +psql (10.6 (Debian 10.6-1.pgdg90+1))
   1.664 +Type "help" for help.
   1.665 +
   1.666 +postgres=#
   1.667 +--END--
   1.668 +
   1.669 +
   1.670 +# -----------------------------------------------------
   1.671 +# List the database tables.
   1.672 +#[user#postgres]
   1.673 +
   1.674 +    \dt
   1.675 +
   1.676 +--START--
   1.677 +                                 List of relations
   1.678 + Schema |                   Name                    | Type  |        Owner
   1.679 +--------+-------------------------------------------+-------+----------------------
   1.680 + public | ft020116adqlcolumnentity                  | table | liengooXien1mooP0tae
   1.681 + public | ft020116adqlresourceentity                | table | liengooXien1mooP0tae
   1.682 + public | ft020116adqlschemaentity                  | table | liengooXien1mooP0tae
   1.683 + public | ft020116adqltableentity                   | table | liengooXien1mooP0tae
   1.684 + public | ft020116authmethodentity                  | table | liengooXien1mooP0tae
   1.685 + public | ft020116bluequeryentity                   | table | liengooXien1mooP0tae
   1.686 + public | ft020116bluequeryentityjointobaseresource | table | liengooXien1mooP0tae
   1.687 + public | ft020116bluetaskentity                    | table | liengooXien1mooP0tae
   1.688 + public | ft020116bluetasklogentity                 | table | liengooXien1mooP0tae
   1.689 + public | ft020116bluetaskparam                     | table | liengooXien1mooP0tae
   1.690 + public | ft020116communityentity                   | table | liengooXien1mooP0tae
   1.691 + public | ft020116configproperty                    | table | liengooXien1mooP0tae
   1.692 + public | ft020116identityentity                    | table | liengooXien1mooP0tae
   1.693 + public | ft020116ivoacolumnentity                  | table | liengooXien1mooP0tae
   1.694 + public | ft020116ivoaresourceentity                | table | liengooXien1mooP0tae
   1.695 + public | ft020116ivoaschemaentity                  | table | liengooXien1mooP0tae
   1.696 + public | ft020116ivoatableentity                   | table | liengooXien1mooP0tae
   1.697 + public | ft020116jdbccolumnentity                  | table | liengooXien1mooP0tae
   1.698 + public | ft020116jdbcresourceentity                | table | liengooXien1mooP0tae
   1.699 + public | ft020116jdbcschemaentity                  | table | liengooXien1mooP0tae
   1.700 + public | ft020116jdbctableentity                   | table | liengooXien1mooP0tae
   1.701 + public | ft020116ogsadqpresourceentity             | table | liengooXien1mooP0tae
   1.702 + public | ft020116ogsaexecresourceentity            | table | liengooXien1mooP0tae
   1.703 + public | ft020116ogsaivoaresourceentity            | table | liengooXien1mooP0tae
   1.704 + public | ft020116ogsajdbcresourceentity            | table | liengooXien1mooP0tae
   1.705 + public | ft020116ogsaserviceentity                 | table | liengooXien1mooP0tae
   1.706 + public | ft020116operationentity                   | table | liengooXien1mooP0tae
   1.707 +(27 rows)
   1.708 +--END--
   1.709 +
   1.710 +# -----------------------------------------------------
   1.711 +# Check the how big the tables are.
   1.712 +#[user#postgres]
   1.713 +
   1.714 +    SELECT COUNT(*) FROM ft020116adqlcolumnentity ;
   1.715 +
   1.716 +--START--
   1.717 +  count
   1.718 +---------
   1.719 + 4431653
   1.720 +--END--
   1.721 +
   1.722 +    SELECT COUNT(*) FROM ft020116adqlschemaentity ;
   1.723 +
   1.724 +--START--
   1.725 + count
   1.726 +--------
   1.727 + 124803
   1.728 +(1 row)
   1.729 +--END--
   1.730 +
   1.731 +    SELECT COUNT(*) FROM ft020116adqltableentity ;
   1.732 +
   1.733 +--START--
   1.734 + count
   1.735 +--------
   1.736 + 166863
   1.737 +(1 row)
   1.738 +--END--
   1.739 +
   1.740 +    SELECT COUNT(*) FROM ft020116authmethodentity ;
   1.741 +
   1.742 +--START--
   1.743 +  count
   1.744 +----------
   1.745 + 16020562
   1.746 +--END--
   1.747 +
   1.748 +    SELECT COUNT(*) FROM ft020116bluequeryentity ;
   1.749 +
   1.750 +--START--
   1.751 + count
   1.752 +--------
   1.753 + 171174
   1.754 +(1 row)
   1.755 +--END--
   1.756 +
   1.757 +    SELECT COUNT(*) FROM ft020116bluequeryentityjointobaseresource ;
   1.758 +
   1.759 +--START--
   1.760 + count
   1.761 +--------
   1.762 + 165455
   1.763 +(1 row)
   1.764 +--END--
   1.765 +
   1.766 +    SELECT COUNT(*) FROM ft020116bluetaskentity ;
   1.767 +
   1.768 +--START--
   1.769 + count
   1.770 +--------
   1.771 + 171174
   1.772 +(1 row)
   1.773 +--END--
   1.774 +
   1.775 +    SELECT COUNT(*) FROM ft020116bluetasklogentity ;
   1.776 +
   1.777 +--START--
   1.778 + count
   1.779 +--------
   1.780 + 552679
   1.781 +(1 row)
   1.782 +--END--
   1.783 +
   1.784 +    SELECT COUNT(*) FROM ft020116bluetaskparam ;
   1.785 +
   1.786 +--START--
   1.787 + count
   1.788 +-------
   1.789 +  2832
   1.790 +(1 row)
   1.791 +--END--
   1.792 +
   1.793 +    SELECT COUNT(*) FROM ft020116communityentity ;
   1.794 +
   1.795 +--START--
   1.796 + count
   1.797 +-------
   1.798 +     2
   1.799 +(1 row)
   1.800 +--END--
   1.801 +
   1.802 +    SELECT COUNT(*) FROM ft020116configproperty ;
   1.803 +
   1.804 +--START--
   1.805 + count
   1.806 +-------
   1.807 +     0
   1.808 +(1 row)
   1.809 +--END--
   1.810 +
   1.811 +    SELECT COUNT(*) FROM ft020116identityentity ;
   1.812 +
   1.813 +--START--
   1.814 +  count
   1.815 +----------
   1.816 + 15313504
   1.817 +(1 row)
   1.818 +--END--
   1.819 +
   1.820 +    SELECT COUNT(*) FROM ft020116ivoacolumnentity ;
   1.821 +
   1.822 +--START--
   1.823 + count
   1.824 +-------
   1.825 +     0
   1.826 +(1 row)
   1.827 +--END--
   1.828 +
   1.829 +    SELECT COUNT(*) FROM ft020116ivoaresourceentity ;
   1.830 +
   1.831 +--START--
   1.832 + count
   1.833 +-------
   1.834 +     0
   1.835 +(1 row)
   1.836 +--END--
   1.837 +
   1.838 +    SELECT COUNT(*) FROM ft020116ivoaschemaentity ;
   1.839 +
   1.840 +--START--
   1.841 + count
   1.842 +-------
   1.843 +     0
   1.844 +(1 row)
   1.845 +--END--
   1.846 +
   1.847 +    SELECT COUNT(*) FROM ft020116ivoatableentity ;
   1.848 +
   1.849 +--START--
   1.850 + count
   1.851 +-------
   1.852 +     0
   1.853 +(1 row)
   1.854 +--END--
   1.855 +
   1.856 +    SELECT COUNT(*) FROM ft020116jdbccolumnentity ;
   1.857 +
   1.858 +--START--
   1.859 +  count
   1.860 +---------
   1.861 + 4431979
   1.862 +(1 row)
   1.863 +--END--
   1.864 +
   1.865 +    SELECT COUNT(*) FROM ft020116jdbcresourceentity ;
   1.866 +
   1.867 +--START--
   1.868 + count
   1.869 +-------
   1.870 +    15
   1.871 +(1 row)
   1.872 +--END--
   1.873 +
   1.874 +    SELECT COUNT(*) FROM ft020116jdbcschemaentity ;
   1.875 +
   1.876 +--START--
   1.877 + count
   1.878 +-------
   1.879 +  1206
   1.880 +(1 row)
   1.881 +--END--
   1.882 +
   1.883 +    SELECT COUNT(*) FROM ft020116jdbctableentity ;
   1.884 +
   1.885 +--START--
   1.886 + count
   1.887 +--------
   1.888 + 167273
   1.889 +(1 row)
   1.890 +--END--
   1.891 +
   1.892 +    SELECT COUNT(*) FROM ft020116ogsadqpresourceentity ;
   1.893 +
   1.894 +--START--
   1.895 + count
   1.896 +-------
   1.897 +     0
   1.898 +(1 row)
   1.899 +--END--
   1.900 +
   1.901 +    SELECT COUNT(*) FROM ft020116ogsaexecresourceentity ;
   1.902 +
   1.903 +--START--
   1.904 + count
   1.905 +--------
   1.906 + 165196
   1.907 +(1 row)
   1.908 +--END--
   1.909 +
   1.910 +    SELECT COUNT(*) FROM ft020116ogsaivoaresourceentity ;
   1.911 +
   1.912 +--START--
   1.913 + count
   1.914 +-------
   1.915 +     0
   1.916 +(1 row)
   1.917 +--END--
   1.918 +
   1.919 +    SELECT COUNT(*) FROM ft020116ogsajdbcresourceentity ;
   1.920 +
   1.921 +--START--
   1.922 + count
   1.923 +-------
   1.924 +     4
   1.925 +(1 row)
   1.926 +--END--
   1.927 +
   1.928 +    SELECT COUNT(*) FROM ft020116ogsaserviceentity ;
   1.929 +
   1.930 +--START--
   1.931 + count
   1.932 +-------
   1.933 +     1
   1.934 +(1 row)
   1.935 +--END--
   1.936 +
   1.937 +    SELECT COUNT(*) FROM ft020116operationentity ;
   1.938 +
   1.939 +--START--
   1.940 +  count
   1.941 +----------
   1.942 + 16020567
   1.943 +(1 row)
   1.944 +--END--
   1.945 +
   1.946 +
   1.947 +# -----------------------------------------------------
   1.948 +# Sort by size.
   1.949 +#[editor]
   1.950 +
   1.951 +    ft020116ogsaserviceentity
   1.952 +         1
   1.953 +
   1.954 +    ft020116communityentity
   1.955 +        2
   1.956 +
   1.957 +    ft020116ogsajdbcresourceentity
   1.958 +         4
   1.959 +
   1.960 +    ft020116jdbcresourceentity
   1.961 +        15
   1.962 +
   1.963 +    ft020116jdbcschemaentity
   1.964 +      1,206
   1.965 +
   1.966 +    ft020116bluetaskparam
   1.967 +      2,832
   1.968 +
   1.969 +    ft020116adqlschemaentity
   1.970 +     124,803
   1.971 +
   1.972 +    ft020116ogsaexecresourceentity
   1.973 +     165,196
   1.974 +
   1.975 +    ft020116bluequeryentityjointobaseresource
   1.976 +     165,455
   1.977 +
   1.978 +    ft020116adqltableentity
   1.979 +     166,863
   1.980 +
   1.981 +    ft020116jdbctableentity
   1.982 +     167,273
   1.983 +
   1.984 +    ft020116bluequeryentity
   1.985 +     171,174
   1.986 +
   1.987 +    ft020116bluetaskentity
   1.988 +     171,174
   1.989 +
   1.990 +    ft020116bluetasklogentity
   1.991 +     552,679
   1.992 +
   1.993 +    ft020116adqlcolumnentity
   1.994 +     4,431,653
   1.995 +
   1.996 +    ft020116jdbccolumnentity
   1.997 +     4,431,979
   1.998 +
   1.999 +    ft020116identityentity
  1.1000 +     15,313,504
  1.1001 +
  1.1002 +    ft020116authmethodentity
  1.1003 +     16,020,562
  1.1004 +
  1.1005 +    ft020116operationentity
  1.1006 +     16,020,567
  1.1007 +
  1.1008 +
  1.1009 +    #
  1.1010 +    # Wayyy too many identities.
  1.1011 +    # - single anon identity
  1.1012 +
  1.1013 +    # Too many authmethodentity
  1.1014 +    # - Can we re-use these ?
  1.1015 +
  1.1016 +    # authmethodentity has way too many fields.
  1.1017 +    # It is just a link from [identity] to [operation] with [method]
  1.1018 +
  1.1019 +--START--
  1.1020 +                 Table "public.ft020116authmethodentity"
  1.1021 +  Column   |            Type             | Collation | Nullable | Default
  1.1022 +-----------+-----------------------------+-----------+----------+---------
  1.1023 + ident     | bigint                      |           | not null |
  1.1024 + created   | timestamp without time zone |           | not null |
  1.1025 + modified  | timestamp without time zone |           | not null |
  1.1026 + uidhi     | bigint                      |           | not null |
  1.1027 + uidlo     | bigint                      |           | not null |
  1.1028 + method    | character varying(255)      |           |          |
  1.1029 + owner     | bigint                      |           |          |
  1.1030 + identity  | bigint                      |           | not null |
  1.1031 + operation | bigint                      |           | not null |
  1.1032 +--END--
  1.1033 +
  1.1034 +    #
  1.1035 +    # Options for optimisation ... but nothing that would kill the service.
  1.1036 +    #
  1.1037 +
  1.1038 +
  1.1039 +# -----------------------------------------------------
  1.1040 +# Check the tapschema database.
  1.1041 +#[user@virtual]
  1.1042 +
  1.1043 +    source chain.properties
  1.1044 +
  1.1045 +    docker exec -it stevedore_carolina_1 \
  1.1046 +        psql \
  1.1047 +            --username "${tapschemauser:?}" \
  1.1048 +            --dbname   "${tapschemadata:?}"
  1.1049 +
  1.1050 +--START--
  1.1051 +psql (10.6 (Debian 10.6-1.pgdg90+1))
  1.1052 +Type "help" for help.
  1.1053 +
  1.1054 +postgres=#
  1.1055 +--END--
  1.1056 +
  1.1057 +# -----------------------------------------------------
  1.1058 +# List the database tables.
  1.1059 +#[user#postgres]
  1.1060 +
  1.1061 +    \dt
  1.1062 +
  1.1063 +--START--
  1.1064 +Did not find any relations.
  1.1065 +--END--
  1.1066 +
  1.1067 +
  1.1068 +# -----------------------------------------------------
  1.1069 +# Check the service properties.
  1.1070 +#[user@virtual]
  1.1071 +
  1.1072 +    grep 'tapschema' chain.properties
  1.1073 +
  1.1074 +--START--
  1.1075 +tapschemadata=postgres
  1.1076 +tapschemauser=ieW7oomei7Laebuu9Ax7
  1.1077 +tapschemapass=vamee3puChoomahz9Iek
  1.1078 +tapschemahost=carolina
  1.1079 +tapschemaport=5432
  1.1080 +tapschematype=pgsql
  1.1081 +tapschemajdbc=tapschemajdbc
  1.1082 +--END--
  1.1083 +
  1.1084 +
  1.1085 +    grep 'tapschema' firethorn.properties
  1.1086 +
  1.1087 +--START--
  1.1088 +firethorn.tapschema.resource.name=tapschemajdbc
  1.1089 +firethorn.tapschema.database.name=postgres
  1.1090 +firethorn.tapschema.database.host=carolina
  1.1091 +firethorn.tapschema.database.port=5432
  1.1092 +firethorn.tapschema.database.user=ieW7oomei7Laebuu9Ax7
  1.1093 +firethorn.tapschema.database.pass=vamee3puChoomahz9Iek
  1.1094 +--END--
  1.1095 +
  1.1096 +
  1.1097 +# -----------------------------------------------------
  1.1098 +# Reboot the virtual machine ....
  1.1099 +#[user@virtual]
  1.1100 +
  1.1101 +    sudo reboot
  1.1102 +
  1.1103 +# -----------------------------------------------------
  1.1104 +# Run the Python client - bringing the dependencies back up.
  1.1105 +#[user@virtual]
  1.1106 +
  1.1107 +    export buildtag=2.1.28
  1.1108 +
  1.1109 +    source "${HOME:?}/chain.properties"
  1.1110 +
  1.1111 +    docker-compose --file "docker-compose.yml" run firethorn-py
  1.1112 +
  1.1113 +--START--
  1.1114 +Starting stevedore_bethany_1  ... done
  1.1115 +Starting stevedore_jarmila_1  ... done
  1.1116 +Starting stevedore_carolina_1 ... done
  1.1117 +Starting stevedore_gillian_1  ... done
  1.1118 +Python 3.5.2 (default, Nov 12 2018, 13:43:14)
  1.1119 +[GCC 5.4.0 20160609] on linux
  1.1120 +Type "help", "copyright", "credits" or "license" for more information.
  1.1121 +--END--
  1.1122 +
  1.1123 +
  1.1124 +# -----------------------------------------------------
  1.1125 +# Login to the container and tail the logs.
  1.1126 +#[user@virtual]
  1.1127 +
  1.1128 +    docker exec -it stevedore_gillian_1 bash
  1.1129 +
  1.1130 +        tail -f logs/firethorn-trace.log
  1.1131 +
  1.1132 +
  1.1133 +# -----------------------------------------------------
  1.1134 +# Run a quick Python test.
  1.1135 +#[python]
  1.1136 +
  1.1137 +import os
  1.1138 +import uuid
  1.1139 +import time
  1.1140 +import firethorn as ftpy
  1.1141 +
  1.1142 +#
  1.1143 +# Create our Firethorn client.
  1.1144 +firethorn = ftpy.Firethorn(
  1.1145 +    endpoint = os.environ.get(
  1.1146 +        'endpoint'
  1.1147 +        )
  1.1148 +    )
  1.1149 +
  1.1150 +#
  1.1151 +# Login using a guest account.
  1.1152 +firethorn.login(
  1.1153 +    str(uuid.uuid4()),
  1.1154 +    str(uuid.uuid4()),
  1.1155 +    None
  1.1156 +    )
  1.1157 +
  1.1158 +resource = firethorn.firethorn_engine.select_adql_resource_by_ident(
  1.1159 +    'http://tap.roe.ac.uk/firethorn/adql/resource/54'
  1.1160 +    )
  1.1161 +
  1.1162 +print(resource)
  1.1163 +--START--
  1.1164 +{
  1.1165 +  "name": "OSA ADQL resource",
  1.1166 +  "fullname": "OSA ADQL resource",
  1.1167 +  "owner": "http://tap.roe.ac.uk/firethorn/community-member/163",
  1.1168 +  "ident": "54",
  1.1169 +  "url": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1170 +  "modified": "2018-12-12T18:44:59.556",
  1.1171 +  "schemas": "http://tap.roe.ac.uk/firethorn/adql/resource/54/schemas/select",
  1.1172 +  "queries": "http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select",
  1.1173 +  "self": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1174 +  "created": "2018-12-12T18:44:59.555",
  1.1175 +  "vosi": "http://tap.roe.ac.uk/firethorn/adql/resource/54/vosi",
  1.1176 +  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
  1.1177 +  "text": null
  1.1178 +}
  1.1179 +--END--
  1.1180 +
  1.1181 +#
  1.1182 +# Create and run a query on the resource.
  1.1183 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
  1.1184 +query_obj = resource.create_query(
  1.1185 +    query_str,
  1.1186 +    "COMPLETED",
  1.1187 +    None,
  1.1188 +    3000000
  1.1189 +    )
  1.1190 +
  1.1191 +print(
  1.1192 +    query_obj
  1.1193 +    )
  1.1194 +
  1.1195 +--START--
  1.1196 +{
  1.1197 +  "columns": [],
  1.1198 +  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1199 +  "results": {
  1.1200 +    "state": "COMPLETED",
  1.1201 +    "count": 1000,
  1.1202 +    "formats": {
  1.1203 +      "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1204 +      "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1205 +    },
  1.1206 +    "table": "http://tap.roe.ac.uk/firethorn/adql/table/57699203"
  1.1207 +  },
  1.1208 +  "modified": "2019-05-09T03:57:24.377",
  1.1209 +  "tables": [],
  1.1210 +  "syntax": {
  1.1211 +    "friendly": null,
  1.1212 +    "status": "VALID",
  1.1213 +    "message": null
  1.1214 +  },
  1.1215 +  "input": "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource",
  1.1216 +  "ident": "57699002",
  1.1217 +  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/blue-query-1.0.json",
  1.1218 +  "history": [
  1.1219 +    {
  1.1220 +      "state": "COMPLETED",
  1.1221 +      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1222 +      "modified": "2019-05-09T03:57:24.327",
  1.1223 +      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1224 +      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698919",
  1.1225 +      "ident": "57699054",
  1.1226 +      "created": "2019-05-09T03:57:24.324",
  1.1227 +      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699054",
  1.1228 +      "level": "INFO",
  1.1229 +      "message": null,
  1.1230 +      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1231 +    },
  1.1232 +    {
  1.1233 +      "state": "RUNNING",
  1.1234 +      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1235 +      "modified": "2019-05-09T03:57:23.988",
  1.1236 +      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1237 +      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698918",
  1.1238 +      "ident": "57699053",
  1.1239 +      "created": "2019-05-09T03:57:23.984",
  1.1240 +      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699053",
  1.1241 +      "level": "INFO",
  1.1242 +      "message": null,
  1.1243 +      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1244 +    },
  1.1245 +    {
  1.1246 +      "state": "READY",
  1.1247 +      "task": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1248 +      "modified": "2019-05-09T03:57:20.823",
  1.1249 +      "self": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1250 +      "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1251 +      "ident": "57699052",
  1.1252 +      "created": "2019-05-09T03:57:20.818",
  1.1253 +      "url": "http://tap.roe.ac.uk/firethorn/blue/log/entry/57699052",
  1.1254 +      "level": "INFO",
  1.1255 +      "message": "Executing query",
  1.1256 +      "type": "http://data.metagrid.co.uk/wfau/firethorn/types/log-entry-1.0.json"
  1.1257 +    }
  1.1258 +  ],
  1.1259 +  "name": "XX_5Y47JQZFGVHTOAAAAFVJVPIFRQ",
  1.1260 +  "text": null,
  1.1261 +  "fields": [],
  1.1262 +  "url": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1263 +  "limits": {
  1.1264 +    "rows": 1000000,
  1.1265 +    "time": null,
  1.1266 +    "cells": null
  1.1267 +  },
  1.1268 +  "delays": {
  1.1269 +    "every": null,
  1.1270 +    "first": null,
  1.1271 +    "last": null
  1.1272 +  },
  1.1273 +  "adql": "SELECT TOP 1000 ra , dec\nFROM ATLASDR1.atlasSource",
  1.1274 +  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/54",
  1.1275 +  "resources": [
  1.1276 +    "http://tap.roe.ac.uk/firethorn/jdbc/resource/53"
  1.1277 +  ],
  1.1278 +  "callback": "http://tap.roe.ac.uk:8081/firethorn/callback/57699002",
  1.1279 +  "mode": "DIRECT",
  1.1280 +  "created": "2019-05-09T03:57:20.140",
  1.1281 +  "osql": "SELECT TOP 1000 [ATLASDR1].[dbo].[atlasSource].[ra] AS [ra] , [ATLASDR1].[dbo].[atlasSource].[dec] AS [dec]\nFROM [ATLASDR1].[dbo].[atlasSource]",
  1.1282 +  "status": "COMPLETED",
  1.1283 +  "self": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1284 +  "workspace": "http://tap.roe.ac.uk/firethorn/adql/resource/54"
  1.1285 +}
  1.1286 +--END--
  1.1287 +
  1.1288 +print(
  1.1289 +    query_obj.table()
  1.1290 +    )
  1.1291 +
  1.1292 +--START--
  1.1293 +{
  1.1294 +  "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1295 +  "fullname": "temp.XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1296 +  "owner": "http://tap.roe.ac.uk/firethorn/community-member/57698915",
  1.1297 +  "name": "XX_GCMJ4MOE5F4LOAAAAFVJVPIIQI",
  1.1298 +  "url": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1299 +  "modified": "2019-05-09T03:57:20.903",
  1.1300 +  "ident": "57699203",
  1.1301 +  "depth": "PARTIAL",
  1.1302 +  "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/57699202",
  1.1303 +  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/57699102",
  1.1304 +  "created": "2019-05-09T03:57:20.903",
  1.1305 +  "formats": {
  1.1306 +    "votable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/votable",
  1.1307 +    "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/datatable"
  1.1308 +  },
  1.1309 +  "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1310 +  "columns": "http://tap.roe.ac.uk/firethorn/adql/table/57699203/columns/select",
  1.1311 +  "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/57699152",
  1.1312 +  "self": "http://tap.roe.ac.uk/firethorn/adql/table/57699203",
  1.1313 +  "query": "http://tap.roe.ac.uk/firethorn/blue/query/57699002",
  1.1314 +  "metadata": {
  1.1315 +    "adql": {
  1.1316 +      "count": 1000,
  1.1317 +      "status": "COMPLETED"
  1.1318 +    }
  1.1319 +  },
  1.1320 +  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
  1.1321 +  "text": null
  1.1322 +}
  1.1323 +--END--
  1.1324 +
  1.1325 +print(
  1.1326 +    query_obj.table().count()
  1.1327 +    )
  1.1328 +
  1.1329 +--START--
  1.1330 +1000
  1.1331 +--END--
  1.1332 +
  1.1333 +#
  1.1334 +# Iterate the metadata tree
  1.1335 +for schema in resource.select_schemas():
  1.1336 +    for table in schema.select_tables():
  1.1337 +        print(
  1.1338 +            "table  [{}][{}][{}]".format(
  1.1339 +                schema.name(),
  1.1340 +                table.name(),
  1.1341 +                table.count(),
  1.1342 +                )
  1.1343 +            )
  1.1344 +        query_str = "SELECT TOP 10 * FROM {}.{}".format(
  1.1345 +            schema.name(),
  1.1346 +            table.name()
  1.1347 +            )
  1.1348 +        query_obj = resource.create_query(
  1.1349 +            query_str,
  1.1350 +            "COMPLETED",
  1.1351 +            None,
  1.1352 +            3000000
  1.1353 +            )
  1.1354 +        py_table = query_obj.table().as_astropy()
  1.1355 +        py_table.pprint()
  1.1356 +
  1.1357 +--START--
  1.1358 +....
  1.1359 +....
  1.1360 +....
  1.1361 +table  [ATLASDR1][CurrentAstrometry][-1]
  1.1362 +Downloading http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable [Done]
  1.1363 +Traceback (most recent call last):
  1.1364 +  File "<stdin>", line 20, in <module>
  1.1365 +  File "/usr/local/lib/python3.5/dist-packages/firethorn/models/adql/adql_table.py", line 97, in as_astropy
  1.1366 +    return astropy_Table.read(self.url + "/votable", format="votable",use_names_over_ids=True)
  1.1367 +  File "/usr/local/lib/python3.5/dist-packages/astropy/table/table.py", line 2548, in read
  1.1368 +    out = io_registry.read(cls, *args, **kwargs)
  1.1369 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/registry.py", line 517, in read
  1.1370 +    data = reader(*args, **kwargs)
  1.1371 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/connect.py", line 73, in read_table_votable
  1.1372 +    input = parse(input, table_id=table_id)
  1.1373 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/table.py", line 137, in parse
  1.1374 +    config=config, pos=(1, 1)).parse(iterator, config)
  1.1375 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3390, in parse
  1.1376 +    iterator, tag, data, config, pos)
  1.1377 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3319, in _add_resource
  1.1378 +    resource.parse(self, iterator, config)
  1.1379 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3141, in parse
  1.1380 +    iterator, tag, data, config, pos)
  1.1381 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 3092, in _add_table
  1.1382 +    table.parse(iterator, config)
  1.1383 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2382, in parse
  1.1384 +    iterator, colnumbers, fields, config)
  1.1385 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2497, in _parse_tabledata
  1.1386 +    vo_reraise(e, config, pos)
  1.1387 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1388 +    raise exc
  1.1389 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2494, in _parse_tabledata
  1.1390 +    fields[i].ID))
  1.1391 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/exceptions.py", line 116, in vo_reraise
  1.1392 +    raise exc
  1.1393 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/tree.py", line 2488, in _parse_tabledata
  1.1394 +    data, config, pos)
  1.1395 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 543, in parse
  1.1396 +    value, mask = parse(x, config, pos)
  1.1397 +  File "/usr/local/lib/python3.5/dist-packages/astropy/io/votable/converters.py", line 815, in parse
  1.1398 +    value = int(value, 10)
  1.1399 +ValueError: http://tap.roe.ac.uk/firethorn/adql/table/57699211/votable:1:21910: ValueError: invalid literal for int() with base 10: '0000000a' (in row 9, col 'column.57699287')
  1.1400 +--END--
  1.1401 +
  1.1402 +
  1.1403 +
  1.1404 +
  1.1405 +# -----------------------------------------------------
  1.1406 +# Service is working and responding to simple queries ...
  1.1407 +#[user@virtual]
  1.1408 +
  1.1409 +    #
  1.1410 +    # Requesting list of queries for the main resource causes an Exception.
  1.1411 +    # http://tap.roe.ac.uk/firethorn/adql/resource/54/queries/select
  1.1412 +
  1.1413 +--START--
  1.1414 +java.lang.OutOfMemoryError: GC overhead limit exceeded
  1.1415 +	org.hibernate.type.descriptor.sql.BigIntTypeDescriptor.getExtractor(BigIntTypeDescriptor.java:60)
  1.1416 +	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:261)
  1.1417 +	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:257)
  1.1418 +	org.hibernate.type.AbstractStandardBasicType.nullSafeGet(AbstractStandardBasicType.java:247)
  1.1419 +	org.hibernate.type.AbstractStandardBasicType.hydrate(AbstractStandardBasicType.java:333)
  1.1420 +	org.hibernate.type.ComponentType.hydrate(ComponentType.java:663)
  1.1421 +	org.hibernate.persister.entity.AbstractEntityPersister.hydrate(AbstractEntityPersister.java:2972)
  1.1422 +	org.hibernate.loader.Loader.loadFromResultSet(Loader.java:1746)
  1.1423 +	org.hibernate.loader.Loader.instanceNotYetLoaded(Loader.java:1672)
  1.1424 +	org.hibernate.loader.Loader.getRow(Loader.java:1561)
  1.1425 +	org.hibernate.loader.Loader.getRowFromResultSet(Loader.java:731)
  1.1426 +	org.hibernate.loader.Loader.processResultSet(Loader.java:990)
  1.1427 +	org.hibernate.loader.Loader.doQuery(Loader.java:948)
  1.1428 +	org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:340)
  1.1429 +	org.hibernate.loader.Loader.doList(Loader.java:2689)
  1.1430 +	org.hibernate.loader.Loader.doList(Loader.java:2672)
  1.1431 +	org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2506)
  1.1432 +	org.hibernate.loader.Loader.list(Loader.java:2501)
  1.1433 +	org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:504)
  1.1434 +	org.hibernate.hql.internal.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:395)
  1.1435 +	org.hibernate.engine.query.spi.HQLQueryPlan.performList(HQLQueryPlan.java:221)
  1.1436 +	org.hibernate.internal.SessionImpl.list(SessionImpl.java:1508)
  1.1437 +	org.hibernate.query.internal.AbstractProducedQuery.doList(AbstractProducedQuery.java:1538)
  1.1438 +	org.hibernate.query.internal.AbstractProducedQuery.list(AbstractProducedQuery.java:1506)
  1.1439 +	uk.ac.roe.wfau.firethorn.entity.AbstractEntityFactory.list(AbstractEntityFactory.java:259)
  1.1440 +	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory.select(BlueQueryEntity.java:535)
  1.1441 +	uk.ac.roe.wfau.firethorn.adql.query.blue.BlueQueryEntity$EntityFactory$$FastClassBySpringCGLIB$$598856d8.invoke(<generated>)
  1.1442 +	org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
  1.1443 +	org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:746)
  1.1444 +	org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
  1.1445 +	org.springframework.transaction.interceptor.TransactionInterceptor$$Lambda$157/756591810.proceedWithInvocation(Unknown Source)
  1.1446 +	org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
  1.1447 +--END--
  1.1448 +
     2.1 --- a/firethorn-webapp/src/main/java/uk/ac/roe/wfau/firethorn/webapp/tap/ScriptRunner.java	Wed May 08 22:40:25 2019 +0100
     2.2 +++ b/firethorn-webapp/src/main/java/uk/ac/roe/wfau/firethorn/webapp/tap/ScriptRunner.java	Thu May 09 05:34:15 2019 +0100
     2.3 @@ -12,6 +12,7 @@
     2.4  
     2.5  /**
     2.6  * Tool to run database scripts
     2.7 +* Original : https://github.com/mybatis/ibatis-2/blob/master/src/main/java/com/ibatis/common/jdbc/ScriptRunner.java
     2.8  */
     2.9  public class ScriptRunner {
    2.10  
    2.11 @@ -228,4 +229,4 @@
    2.12             errorLogWriter.flush();
    2.13         }
    2.14     }
    2.15 -}
    2.16 \ No newline at end of file
    2.17 +}