firethorn

changeset 4259:a04688b9e85f

Some more notes on testing parallel sync validators
author Stelios <stv@roe.ac.uk>
date Tue Jun 25 16:06:03 2019 +0100 (20 months ago)
parents 466baa168aa9
children 9d2e48336608
files doc/notes/stv/20190625-Testing-TAP-parallel.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190625-Testing-TAP-parallel.txt	Tue Jun 25 16:06:03 2019 +0100
     1.3 @@ -0,0 +1,232 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +
    1.27 +
    1.28 +## Test: Run  30 TAP Validators in Parallel
    1.29 +## VM: Arraybwyn
    1.30 +## tap.roe.ac.uk
    1.31 +## Firethorn Version: 2.1.33 
    1.32 +
    1.33 +## Causes the Firethorn Container to stop responding, but not consistently
    1.34 +
    1.35 +python3
    1.36 +
    1.37 +import os                                                                       
    1.38 +from multiprocessing import Pool                                                
    1.39 +
    1.40 +process_list = range(100)
    1.41 +	
    1.42 +def run_process(process_counter):                                                             
    1.43 +    print ("Starting RUN # " + str(process_counter))
    1.44 +    os.system('python3 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://tap.roe.ac.uk/firethorn -r=60 -u= -p= -g= -m=sync')                                       
    1.45 +    print ("Ended RUN # " + str(process_counter))
    1.46 +
    1.47 +
    1.48 +pool = Pool(processes=30)                                                        
    1.49 +pool.map(run_process, process_list)   
    1.50 +
    1.51 +..
    1.52 +
    1.53 +## Test started around: 2019-06-25 10:55:00,000:
    1.54 +## Lost connection to VM at 2019-06-25 11:36:28,361
    1.55 +
    1.56 +## Terminal that was following the firethorn-error log failed with: packet_write_wait: Connection to UNKNOWN port 65535: Broken pipe
    1.57 +
    1.58 +2019-06-25 11:36:28,181 ERROR [FireThornTaskExecutor-85] [OgsaBaseResourceEntity] Ping test failed [27564578][DataRequestExecutionResource][404] 
    1.59 +2019-06-25 11:36:28,361 ERROR [FireThornTaskExecutor-53] [OgsaBaseResourceEntity] Ping test failed [27564579][DataRequestExecutionResource][404] 
    1.60 +packet_write_wait: Connection to UNKNOWN port 65535: Broken pipe
    1.61 +
    1.62 +
    1.63 +## stelios@stelios-pc:~$ ssh Stevedore@Araybwyn 
    1.64 +## Does not respond
    1.65 +## curl http://tap.roe.ac.uk/firethorn/system/info does not respond either
    1.66 +
    1.67 +
    1.68 +## At the time where ssh connection was lost, all table queries where failing:
    1.69 +
    1.70 +Testing [BestDR8.PhotoFamily]
    1.71 +Testing [BestDR8.Dependency]
    1.72 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR8.Dependency&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
    1.73 +http://tap.roe.ac.uk/firethorn/tap/60/sync?QUERY=SELECT+TOP+5+%2A+FROM+BestDR8.PhotoFamily&REQUEST=doQuery&LANG=ADQL&FORMAT=VOTABLE
    1.74 +ERROR:root:<urlopen error timed out>
    1.75 +Traceback (most recent call last):
    1.76 +  File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
    1.77 +    encode_chunked=req.has_header('Transfer-encoding'))
    1.78 +  File "/usr/lib/python3.6/http/client.py", line 1239, in request
    1.79 +    self._send_request(method, url, body, headers, encode_chunked)
    1.80 +  File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
    1.81 +    self.endheaders(body, encode_chunked=encode_chunked)
    1.82 +  File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
    1.83 +    self._send_output(message_body, encode_chunked=encode_chunked)
    1.84 +  File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
    1.85 +    self.send(msg)
    1.86 +  File "/usr/lib/python3.6/http/client.py", line 964, in send
    1.87 +    self.connect()
    1.88 +  File "/usr/lib/python3.6/http/client.py", line 936, in connect
    1.89 +    (self.host,self.port), self.timeout, self.source_address)
    1.90 +  File "/usr/lib/python3.6/socket.py", line 724, in create_connection
    1.91 +    raise err
    1.92 +  File "/usr/lib/python3.6/socket.py", line 713, in create_connection
    1.93 +    sock.connect(sa)
    1.94 +socket.timeout: timed out
    1.95 +
    1.96 +During handling of the above exception, another exception occurred:
    1.97 +
    1.98 +Traceback (most recent call last):
    1.99 +  File "/usr/local/lib/python3.6/dist-packages/firethorn-0.1.0-py3.6.egg/firethorn/tap/voQuery.py", line 133, in execute_sync_query
   1.100 +    self.votable = Table.read(full_url, format="votable")
   1.101 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/table/connect.py", line 52, in __call__
   1.102 +    out = registry.read(cls, *args, **kwargs)
   1.103 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/registry.py", line 523, in read
   1.104 +    data = reader(*args, **kwargs)
   1.105 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/connect.py", line 73, in read_table_votable
   1.106 +    input = parse(input, table_id=table_id)
   1.107 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/io/votable/table.py", line 135, in parse
   1.108 +    _debug_python_based_parser=_debug_python_based_parser) as iterator:
   1.109 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   1.110 +    return next(self.gen)
   1.111 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 157, in get_xml_iterator
   1.112 +    with _convert_to_fd_or_read_function(source) as fd:
   1.113 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   1.114 +    return next(self.gen)
   1.115 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/xml/iterparser.py", line 63, in _convert_to_fd_or_read_function
   1.116 +    with data.get_readable_fileobj(fd, encoding='binary') as new_fd:
   1.117 +  File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
   1.118 +    return next(self.gen)
   1.119 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 192, in get_readable_fileobj
   1.120 +    timeout=remote_timeout)
   1.121 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1091, in download_file
   1.122 +    raise e
   1.123 +  File "/usr/local/lib/python3.6/dist-packages/astropy-3.2.1-py3.6-linux-x86_64.egg/astropy/utils/data.py", line 1024, in download_file
   1.124 +    with urllib.request.urlopen(remote_url, timeout=timeout) as remote:
   1.125 +  File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
   1.126 +    return opener.open(url, data, timeout)
   1.127 +  File "/usr/lib/python3.6/urllib/request.py", line 526, in open
   1.128 +    response = self._open(req, data)
   1.129 +  File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
   1.130 +    '_open', req)
   1.131 +  File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
   1.132 +    result = func(*args)
   1.133 +  File "/usr/lib/python3.6/urllib/request.py", line 1346, in http_open
   1.134 +    return self.do_open(http.client.HTTPConnection, req)
   1.135 +  File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
   1.136 +    raise URLError(err)
   1.137 +urllib.error.URLError: <urlopen error timed out>
   1.138 +Rowcount:-1
   1.139 +<class 'Exception'>
   1.140 +{'BestDR1.Field': '<urlopen error timed out>', 'BestDR1.SpecLineAll': '<urlopen error timed out>', 'BestDR1.DataConstants': '<urlopen error timed out>', 'BestDR1.TilingNote': '<urlopen error timed out>', 'BestDR1.SiteDiagnostics': '<urlopen error timed out>', 'BestDR1.PhotoObjAll': '<urlopen error timed out>', 'BestDR1.Frame': '<urlopen error timed out>', 'BestDR1."Zone"': '<urlopen error timed out>', 'BestDR1.PartitionMap': '<urlopen error timed out>', 'BestDR1.DBObjects': '<urlopen error timed out>', 'BestDR1.Sector': '<urlopen error timed out>', 'BestDR1.DBObjectDescription': '<urlopen error timed out>', 'BestDR1.Glossary': '<urlopen error timed out>', 'BestDR1.TilingRun': '<urlopen error timed out>', 'BestDR1.Chunk': '<urlopen error timed out>', 'BestDR1.Segment': '<urlopen error timed out>', 'BestDR1."First"': '<urlopen error timed out>', 'BestDR1.PlateX': '<urlopen error timed out>', 'BestDR1.PubHistory': '<urlopen error timed out>', 'BestDR1.FieldProfile': '<urlopen error timed out>', 'BestDR1.Best2Sector': '<urlopen error timed out>', 'BestDR1.DBViewCols': '<urlopen error timed out>', 'BestDR1.Mask': '<urlopen error timed out>', 'BestDR1.Target': '<urlopen error timed out>', 'BestDR1.IndexMap': '<urlopen error timed out>', 'BestDR1.TiledTargetAll': '<urlopen error timed out>', 'BestDR1.TargetInfo': '<urlopen error timed out>', 'BestDR1.Sector2Tile': '<urlopen error timed out>', 'BestDR1.SpecPhotoAll': '<urlopen error timed out>', 'BestDR3.Algorithm': '<urlopen error timed out>', 'BestDR3.Field': '<urlopen error timed out>', 'BestDR3.FieldProfile': '<urlopen error timed out>', 'BestDR3.FileGroupMap': '<urlopen error timed out>', 'BestDR3."First"': '<urlopen error timed out>', 'BestDR3.Frame': '<urlopen error timed out>', 'BestDR3.Glossary': '<urlopen error timed out>', 'BestDR3.History': '<urlopen error timed out>', 'BestDR3.HoleObj': '<urlopen error timed out>', 'BestDR3.IndexMap': '<urlopen error timed out>', 'BestDR3.LoadHistory': '<urlopen error timed out>', 'BestDR3.Mask': '<urlopen error timed out>', 'BestDR3.MaskedObject': '<urlopen error timed out>', 'BestDR3."Match"': '<urlopen error timed out>', 'BestDR3.MatchHead': '<urlopen error timed out>', 'BestDR3.PhotoObjAll': '<urlopen error timed out>', 'BestDR3.PhotoProfile': '<urlopen error timed out>', 'BestDR3.PhotoTag': '<urlopen error timed out>', 'BestDR3.Photoz': '<urlopen error timed out>', 'BestDR3.PlateX': '<urlopen error timed out>', 'BestDR3.ProfileDefs': '<urlopen error timed out>', 'BestDR3.PubHistory': '<urlopen error timed out>', 'BestDR3.QsoCatalogAll': '<urlopen error timed out>', 'BestDR3.QsoConcordance': '<urlopen error timed out>', 'BestDR3.QueryResults': '<urlopen error timed out>', 'BestDR3.RC3': '<urlopen error timed out>', 'BestDR3.RecentQueries': '<urlopen error timed out>', 'BestDR3."Region"': '<urlopen error timed out>', 'BestDR3.Rmatrix': '<urlopen error timed out>', 'BestDR3.Rosat': '<urlopen error timed out>', 'BestDR3.RunQA': '<urlopen error timed out>', 'BestDR3.RunShift': '<urlopen error timed out>', 'BestDR3.SDSSConstants': '<urlopen error timed out>', 'BestDR3.Sector': '<urlopen error timed out>', 'BestDR3.SpecObj': '<urlopen error timed out>', 'BestDR3.PhotoFamily': '<urlopen error timed out>', 'BestDR3.PhotoObj': '<urlopen error timed out>', 'BestDR3.PhotoPrimary': '<urlopen error timed out>', 'BestDR3.PhotoSecondary': '<urlopen error timed out>', 'BestDR3.SpecLine': '<urlopen error timed out>', 'BestDR3.SpecPhoto': '<urlopen error timed out>', 'BestDR3.Galaxy': '<urlopen error timed out>', 'BestDR3.Sky': '<urlopen error timed out>', 'BestDR3.Star': '<urlopen error timed out>', 'BestDR3.Unknown': '<urlopen error timed out>', 'BestDR5.SpecObjAll': '<urlopen error timed out>', 'BestDR5.TilingInfo': '<urlopen error timed out>', 'BestDR5.QsoBunch': '<urlopen error timed out>', 'BestDR5.HoleType': '<urlopen error timed out>', 'BestDR5.PhotoStatus': '<urlopen error timed out>', 'BestDR5.SDSSConstants': '<urlopen error timed out>', 'BestDR5.SiteDiagnostics': '<urlopen error timed out>', 'BestDR5.PubHistory': '<urlopen error timed out>', 'BestDR5.MaskType': '<urlopen error timed out>', 'BestDR5.PhotoPrimary': '<urlopen error timed out>', 'BestDR5.History': '<urlopen error timed out>', 'BestDR5.SpecLineIndex': '<urlopen error timed out>', 'BestDR5.Sector': '<urlopen error timed out>', 'BestDR5.PhotoFlags': '<urlopen error timed out>', 'BestDR5.FieldQuality': '<urlopen error timed out>', 'BestDR5.SpecObj': '<urlopen error timed out>', 'BestDR5.PhotoSecondary': '<urlopen error timed out>', 'BestDR5.TilingRun': '<urlopen error timed out>', 'BestDR5.PhotoAux': '<urlopen error timed out>', 'BestDR5.RunShift': '<urlopen error timed out>', 'BestDR5.FieldProfile': '<urlopen error timed out>', 'BestDR5.Frame': '<urlopen error timed out>', 'BestDR5.Region2Box': '<urlopen error timed out>', 'BestDR5.Sky': '<urlopen error timed out>', 'BestDR5.TilingBoundary': '<urlopen error timed out>', 'BestDR5.FramesStatus': '<urlopen error timed out>', 'BestDR5.StripeDefs': '<urlopen error timed out>', 'BestDR5.TableDesc': '<urlopen error timed out>', 'BestDR5.QsoTarget': '<urlopen error timed out>', 'BestDR5.FieldMask': '<urlopen error timed out>', 'BestDR5.TargetParam': '<urlopen error timed out>', 'BestDR5.QsoCatalog': '<urlopen error timed out>', 'BestDR5.Photoz': '<urlopen error timed out>', 'BestDR5."Match"': '<urlopen error timed out>', 'BestDR5.SiteConstants': '<urlopen error timed out>', 'BestDR5.PhotoMode': '<urlopen error timed out>', 'BestDR5."Region"': '<urlopen error timed out>', 'BestDR5.Chunk': '<urlopen error timed out>', 'BestDR5.SpecLine': '<urlopen error timed out>', 'BestDR5.CoordType': '<urlopen error timed out>', 'BestDR5.SpecLineAll': '<urlopen error timed out>', 'BestDR5.StarTag': '<urlopen error timed out>', 'BestDR5.PspStatus': '<urlopen error timed out>', 'BestDR5.Versions': '<urlopen error timed out>', 'BestDR5.BestTarget2Sector': '<urlopen error timed out>', 'BestDR5.Inventory': '<urlopen error timed out>', 'BestDR5.Rmatrix': '<urlopen error timed out>', 'BestDR5.TilingGeometry': '<urlopen error timed out>', 'BestDR5.Segment': '<urlopen error timed out>', 'BestDR5.LoadHistory': '<urlopen error timed out>', 'BestDR5.ProperMotions': '<urlopen error timed out>', 'BestDR5.IndexMap': '<urlopen error timed out>', 'BestDR5.Mask': '<urlopen error timed out>', 'BestDR5.Target': '<urlopen error timed out>', 'BestDR5.SpecPhotoAll': '<urlopen error timed out>', 'BestDR5.DBObjects': '<urlopen error timed out>', 'BestDR5.Neighbors': '<urlopen error timed out>', 'BestDR5.DBColumns': '<urlopen error timed out>', 'BestDR5.ObjMask': '<urlopen error timed out>', 'BestDR5.TilingMask': '<urlopen error timed out>', 'BestDR5.RecentQueries': '<urlopen error timed out>', 'BestDR5.PrimTarget': '<urlopen error timed out>', 'BestDR5.Galaxy': '<urlopen error timed out>', 'BestDR5.MaskedObject': '<urlopen error timed out>', 'BestDR5.MatchHead': '<urlopen error timed out>', 'BestDR5.PhotoProfile': '<urlopen error timed out>', 'BestDR5.USNO': '<urlopen error timed out>', 'BestDR5.TileAll': '<urlopen error timed out>', 'BestDR5.DBViewCols': '<urlopen error timed out>', 'BestDR5.PartitionMap': '<urlopen error timed out>', 'BestDR5.FileGroupMap': '<urlopen error timed out>', 'BestDR5.Glossary': '<urlopen error timed out>', 'BestDR5."Diagnostics"': '<urlopen error timed out>', 'BestDR5.Field': '<urlopen error timed out>', 'BestDR5.TargetInfo': '<urlopen error timed out>', 'BestDR5.DataConstants': '<urlopen error timed out>', 'BestDR5.TilingNote': '<urlopen error timed out>', 'BestDR5.RegionArcs': '<urlopen error timed out>', 'BestDR5.Stetson': '<urlopen error timed out>', 'BestDR5.GalaxyTag': '<urlopen error timed out>', 'BestDR5.Run': '<urlopen error timed out>', 'BestDR5.PhotoAuxAll': '<urlopen error timed out>', 'BestDR5."First"': '<urlopen error timed out>', 'BestDR5.TargRunQA': '<urlopen error timed out>', 'BestDR5.HoleObj': '<urlopen error timed out>', 'BestDR5.QuasarCatalog': '<urlopen error timed out>', 'BestDR5.SpecZStatus': '<urlopen error timed out>', 'BestDR5.PhotoTag': '<urlopen error timed out>', 'BestDR5.Columns': '<urlopen error timed out>', 'BestDR5.SpecLineNames': '<urlopen error timed out>', 'BestDR5.SiteDBs': '<urlopen error timed out>', 'BestDR5.TiledTarget': '<urlopen error timed out>', 'BestDR5.Star': '<urlopen error timed out>', 'BestDR5.QueryResults': '<urlopen error timed out>', 'BestDR5.SpecZWarning': '<urlopen error timed out>', 'BestDR5.QsoCatalogAll': '<urlopen error timed out>', 'BestDR5.RegionConvex': '<urlopen error timed out>', 'BestDR5.Algorithm': '<urlopen error timed out>', 'BestDR5.Photoz2': '<urlopen error timed out>', 'BestDR5."Zone"': '<urlopen error timed out>', 'BestDR5.ImageMask': '<urlopen error timed out>', 'BestDR5.QsoConcordanceAll': '<urlopen error timed out>', 'BestDR5.ProgramType': '<urlopen error timed out>', 'BestDR5.SpecClass': '<urlopen error timed out>', 'BestDR5.PhotoObjAll': '<urlopen error timed out>', 'BestDR5.RC3': '<urlopen error timed out>', 'BestDR5.PhotoType': '<urlopen error timed out>', 'BestDR5.Tile': '<urlopen error timed out>', 'BestDR5.HalfSpace': '<urlopen error timed out>', 'BestDR5.PlateX': '<urlopen error timed out>', 'BestDR5.QsoBest': '<urlopen error timed out>', 'BestDR5.QsoSpec': '<urlopen error timed out>', 'BestDR5.SpecPhoto': '<urlopen error timed out>', 'BestDR5.Unknown': '<urlopen error timed out>', 'BestDR5.RunQA': '<urlopen error timed out>', 'BestDR5.PhotoFamily': '<urlopen error timed out>', 'BestDR5.TiledTargetAll': '<urlopen error timed out>', 'BestDR5.Dependency': '<urlopen error timed out>', 'BestDR5.PhotoObj': '<urlopen error timed out>', 'BestDR5.InsideMask': '<urlopen error timed out>', 'BestDR5.Sector2Tile': '<urlopen error timed out>', 'BestDR7.SpecObjAll': '<urlopen error timed out>', 'BestDR7.TilingInfo': '<urlopen error timed out>', 'BestDR7.QsoBunch': '<urlopen error timed out>', 'BestDR7.Ap7Mag': '<urlopen error timed out>', 'BestDR7.SDSSConstants': '<urlopen error timed out>', 'BestDR7.TargPhotoObj': '<urlopen error timed out>', 'BestDR7.PubHistory': '<urlopen error timed out>', 'BestDR7.QsoConcordance': '<urlopen error timed out>', 'BestDR7.MaskType': '<urlopen error timed out>', 'BestDR7.ProfileDefs': '<urlopen error timed out>', 'BestDR7.Rosat': '<urlopen error timed out>', 'BestDR7.TiMask': '<urlopen error timed out>', 'BestDR7.ELRedShift': '<urlopen error timed out>', 'BestDR7.SecTarget': '<urlopen error timed out>', 'BestDR7.XCRedshift': '<urlopen error timed out>', 'BestDR7.ObjType': '<urlopen error timed out>', 'BestDR7.PhotoPrimary': '<urlopen error timed out>', 'BestDR7.History': '<urlopen error timed out>', 'BestDR7."Zone"': '<urlopen error timed out>', 'BestDR7.Sector': '<urlopen error timed out>', 'BestDR7.PhotoFlags': '<urlopen error timed out>', 'BestDR7.FieldQuality': '<urlopen error timed out>', 'BestDR7.SpecObj': '<urlopen error timed out>', 'BestDR7.PhotoSecondary': '<urlopen error timed out>', 'BestDR7.TilingRun': '<urlopen error timed out>', 'BestDR7.TilingMask': '<urlopen error timed out>', 'BestDR7.PhotoAux': '<urlopen error timed out>', 'BestDR7.TargPhotoSecondary': '<urlopen error timed out>', 'BestDR7.RunShift': '<urlopen error timed out>', 'BestDR7.FieldProfile': '<urlopen error timed out>', 'BestDR7.Frame': '<urlopen error timed out>', 'BestDR7.Region2Box': '<urlopen error timed out>', 'BestDR7.Sky': '<urlopen error timed out>', 'BestDR7.TilingBoundary': '<urlopen error timed out>', 'BestDR7.FramesStatus': '<urlopen error timed out>', 'BestDR7.StripeDefs': '<urlopen error timed out>', 'BestDR7.TableDesc': '<urlopen error timed out>', 'BestDR7.QsoTarget': '<urlopen error timed out>', 'BestDR7.UberAstro': '<urlopen error timed out>', 'BestDR7.FieldMask': '<urlopen error timed out>', 'BestDR7.TargetParam': '<urlopen error timed out>', 'BestDR7.QsoCatalog': '<urlopen error timed out>', 'BestDR7.Photoz': '<urlopen error timed out>', 'BestDR7."Match"': '<urlopen error timed out>', 'BestDR7.SiteConstants': '<urlopen error timed out>', 'BestDR7.PhotoMode': '<urlopen error timed out>', 'BestDR7."Region"': '<urlopen error timed out>', 'BestDR7.Chunk': '<urlopen error timed out>', 'BestDR7.SpecLine': '<urlopen error timed out>', 'BestDR7.CoordType': '<urlopen error timed out>', 'BestDR7.UberCalibStatus': '<urlopen error timed out>', 'BestDR7.SpecLineAll': '<urlopen error timed out>', 'BestDR7.StarTag': '<urlopen error timed out>', 'BestDR7.PspStatus': '<urlopen error timed out>', 'BestDR7.PsObjAll': '<urlopen error timed out>', 'BestDR7.BestTarget2Sector': '<urlopen error timed out>', 'BestDR7.Inventory': '<urlopen error timed out>', 'BestDR7.Rmatrix': '<urlopen error timed out>', 'BestDR7.TilingGeometry': '<urlopen error timed out>', 'BestDR7.Versions': '<urlopen error timed out>', 'BestDR7.Segment': '<urlopen error timed out>', 'BestDR7.LoadHistory': '<urlopen error timed out>', 'BestDR7.ProperMotions': '<urlopen error timed out>', 'BestDR7.IndexMap': '<urlopen error timed out>', 'BestDR7.Mask': '<urlopen error timed out>', 'BestDR7.Target': '<urlopen error timed out>', 'BestDR7.SpecPhotoAll': '<urlopen error timed out>', 'BestDR7.DBObjects': '<urlopen error timed out>', 'BestDR7.Neighbors': '<urlopen error timed out>', 'BestDR7.DBColumns': '<urlopen error timed out>', 'BestDR7.ObjMask': '<urlopen error timed out>', 'BestDR7.DR3QuasarCatalog': '<urlopen error timed out>', 'BestDR7.RecentQueries': '<urlopen error timed out>', 'BestDR7.PrimTarget': '<urlopen error timed out>', 'BestDR7.TargPhotoObjAll': '<urlopen error timed out>', 'BestDR7.Galaxy': '<urlopen error timed out>', 'BestDR7.MaskedObject': '<urlopen error timed out>', 'BestDR7.MatchHead': '<urlopen error timed out>', 'BestDR7.PhotoProfile': '<urlopen error timed out>', 'BestDR7.TargPhotoPrimary': '<urlopen error timed out>', 'BestDR7.USNO': '<urlopen error timed out>', 'BestDR7.TileAll': '<urlopen error timed out>', 'BestDR7.DBViewCols': '<urlopen error timed out>', 'BestDR7.PartitionMap': '<urlopen error timed out>', 'BestDR7.FileGroupMap': '<urlopen error timed out>', 'BestDR7.Glossary': '<urlopen error timed out>', 'BestDR7."Diagnostics"': '<urlopen error timed out>', 'BestDR7.Field': '<urlopen error timed out>', 'BestDR7.TargetInfo': '<urlopen error timed out>', 'BestDR7.DataConstants': '<urlopen error timed out>', 'BestDR7.TilingNote': '<urlopen error timed out>', 'BestDR7.sppParams': '<urlopen error timed out>', 'BestDR7.RegionArcs': '<urlopen error timed out>', 'BestDR7.Stetson': '<urlopen error timed out>', 'BestDR7.GalaxyTag': '<urlopen error timed out>', 'BestDR7.FieldQA': '<urlopen error timed out>', 'BestDR7.Run': '<urlopen error timed out>', 'BestDR7.PhotoAuxAll': '<urlopen error timed out>', 'BestDR7."First"': '<urlopen error timed out>', 'BestDR7.DR5QuasarCatalog': '<urlopen error timed out>', 'BestDR7.TargPhotoTag': '<urlopen error timed out>', 'BestDR7.RegionPatch': '<urlopen error timed out>', 'BestDR7.TargRunQA': '<urlopen error timed out>', 'BestDR7.spbsParams': '<urlopen error timed out>', 'BestDR7.HoleObj': '<urlopen error timed out>', 'BestDR7.SpecZStatus': '<urlopen error timed out>', 'BestDR7.PhotoTag': '<urlopen error timed out>', 'BestDR7.Columns': '<urlopen error timed out>', 'BestDR7.SpecLineNames': '<urlopen error timed out>', 'BestDR7.SiteDBs': '<urlopen error timed out>', 'BestDR7.TiledTarget': '<urlopen error timed out>', 'BestDR7.UberCal': '<urlopen error timed out>', 'BestDR7.Star': '<urlopen error timed out>', 'BestDR7.QueryResults': '<urlopen error timed out>', 'BestDR7.SpecZWarning': '<urlopen error timed out>', 'BestDR7.QsoCatalogAll': '<urlopen error timed out>', 'BestDR7.OrigField': '<urlopen error timed out>', 'BestDR7.RegionConvex': '<urlopen error timed out>', 'BestDR7.Algorithm': '<urlopen error timed out>', 'BestDR7.OrigPhotoObjAll': '<urlopen error timed out>', 'BestDR7.Photoz2': '<urlopen error timed out>', 'BestDR7.sppLines': '<urlopen error timed out>', 'BestDR7.SpecLineIndex': '<urlopen error timed out>', 'BestDR7.ImageMask': '<urlopen error timed out>', 'BestDR7.QsoConcordanceAll': '<urlopen error timed out>', 'BestDR7.ProgramType': '<urlopen error timed out>', 'BestDR7.SpecClass': '<urlopen error timed out>', 'BestDR7.PhotoObjAll': '<urlopen error timed out>', 'BestDR7.RC3': '<urlopen error timed out>', 'BestDR7.PhotoType': '<urlopen error timed out>', 'BestDR7.Tile': '<urlopen error timed out>', 'BestDR7.HalfSpace': '<urlopen error timed out>', 'BestDR7.PlateX': '<urlopen error timed out>', 'BestDR7.QsoBest': '<urlopen error timed out>', 'BestDR7.QsoSpec': '<urlopen error timed out>', 'BestDR7.SpecPhoto': '<urlopen error timed out>', 'BestDR7.Unknown': '<urlopen error timed out>', 'BestDR7.RunQA': '<urlopen error timed out>', 'BestDR7.PhotoFamily': '<urlopen error timed out>', 'BestDR7.TiledTargetAll': '<urlopen error timed out>', 'BestDR7.Dependency': '<urlopen error timed out>', 'BestDR7.PhotoObj': '<urlopen error timed out>', 'BestDR7.InsideMask': '<urlopen error timed out>', 'BestDR7.Sector2Tile': '<urlopen error timed out>', 'BestDR8.SpecObjAll': '<urlopen error timed out>', 'BestDR8.TwoMass': '<urlopen error timed out>', 'BestDR8.FieldMask': '<urlopen error timed out>', 'BestDR8.HoleType': '<urlopen error timed out>', 'BestDR8.PhotoStatus': '<urlopen error timed out>', 'BestDR8.SDSSConstants': '<urlopen error timed out>', 'BestDR8.galSpecIndx': '<urlopen error timed out>', 'BestDR8.ObjType': '<urlopen error timed out>', 'BestDR8.sdssTilingGeometry': '<urlopen error timed out>', 'BestDR8.PhotoPrimary': '<urlopen error timed out>', 'BestDR8.History': '<urlopen error timed out>', 'BestDR8.Plate2Target': '<urlopen error timed out>', 'BestDR8.detectionIndex': '<urlopen error timed out>', 'BestDR8.segue1SpecObjAll': '<urlopen error timed out>', 'BestDR8.thingIndex': '<urlopen error timed out>', 'BestDR8.Sector': '<urlopen error timed out>', 'BestDR8.PhotoFlags': '<urlopen error timed out>', 'BestDR8.FieldQuality': '<urlopen error timed out>', 'BestDR8.SpecObj': '<urlopen error timed out>', 'BestDR8.PhotoSecondary': '<urlopen error timed out>', 'BestDR8.RunShift': '<urlopen error timed out>', 'BestDR8.FieldProfile': '<urlopen error timed out>', 'BestDR8.zooSpec': '<urlopen error timed out>', 'BestDR8.Region2Box': '<urlopen error timed out>', 'BestDR8.Sky': '<urlopen error timed out>', 'BestDR8.TiMask': '<urlopen error timed out>', 'BestDR8.StripeDefs': '<urlopen error timed out>', 'BestDR8.sdssTilingBoundary': '<urlopen error timed out>', 'BestDR8.sdssTileAll': '<urlopen error timed out>', 'BestDR8.PhotoPrimaryDR7': '<urlopen error timed out>', 'BestDR8.segueTargetAll': '<urlopen error timed out>', 'BestDR8.Photoz': '<urlopen error timed out>', 'BestDR8.galSpecInfo': '<urlopen error timed out>', 'BestDR8.SiteConstants': '<urlopen error timed out>', 'BestDR8.PhotoMode': '<urlopen error timed out>', 'BestDR8."Region"': '<urlopen error timed out>', 'BestDR8.Frame': '<urlopen error timed out>', 'BestDR8.PhotozTemplateCoeff': '<urlopen error timed out>', 'BestDR8.CoordType': '<urlopen error timed out>', 'BestDR8.zooVotes': '<urlopen error timed out>', 'BestDR8.UberCalibStatus': '<urlopen error timed out>', 'BestDR8.zooConfidence': '<urlopen error timed out>', 'BestDR8.zooMirrorBias': '<urlopen error timed out>', 'BestDR8.PspStatus': '<urlopen error timed out>', 'BestDR8.Versions': '<urlopen error timed out>', 'BestDR8.StarTag': '<urlopen error timed out>', 'BestDR8.PhotoType': '<urlopen error timed out>', 'BestDR8.Inventory': '<urlopen error timed out>', 'BestDR8.Rmatrix': '<urlopen error timed out>', 'BestDR8.LoadHistory': '<urlopen error timed out>', 'BestDR8.PhotozRFTemplateCoeff': '<urlopen error timed out>', 'BestDR8.neighbors': '<urlopen error timed out>', 'BestDR8.sdssTilingMask': '<urlopen error timed out>', 'BestDR8.Mask': '<urlopen error timed out>', 'BestDR8.sdssTiledTargetAll': '<urlopen error timed out>', 'BestDR8.FramesStatus': '<urlopen error timed out>', 'BestDR8.Target': '<urlopen error timed out>', 'BestDR8.SpecPhotoAll': '<urlopen error timed out>', 'BestDR8.DBObjects': '<urlopen error timed out>', 'BestDR8.DBColumns': '<urlopen error timed out>', 'BestDR8.RecentQueries': '<urlopen error timed out>', 'BestDR8.PrimTarget': '<urlopen error timed out>', 'BestDR8.TwoMassXSC': '<urlopen error timed out>', 'BestDR8.Galaxy': '<urlopen error timed out>', 'BestDR8.MaskedObject': '<urlopen error timed out>', 'BestDR8.PhotoProfile': '<urlopen error timed out>', 'BestDR8.USNO': '<urlopen error timed out>', 'BestDR8.TableDesc': '<urlopen error timed out>', 'BestDR8.DBViewCols': '<urlopen error timed out>', 'BestDR8.ROSAT': '<urlopen error timed out>', 'BestDR8.FileGroupMap': '<urlopen error timed out>', 'BestDR8.sdssImagingHalfSpaces': '<urlopen error timed out>', 'BestDR8.sdssPolygon2Field': '<urlopen error timed out>', 'BestDR8."Diagnostics"': '<urlopen error timed out>', 'BestDR8.Field': '<urlopen error timed out>', 'BestDR8.TargetInfo': '<urlopen error timed out>', 'BestDR8.DataConstants': '<urlopen error timed out>', 'BestDR8.sdssTargetParam': '<urlopen error timed out>', 'BestDR8.RegionArcs': '<urlopen error timed out>', 'BestDR8.GalaxyTag': '<urlopen error timed out>', 'BestDR8.galSpecLine': '<urlopen error timed out>', 'BestDR8.Run': '<urlopen error timed out>', 'BestDR8.ProperMotions': '<urlopen error timed out>', 'BestDR8.galSpecExtra': '<urlopen error timed out>', 'BestDR8.RegionPatch': '<urlopen error timed out>', 'BestDR8.IndexMap': '<urlopen error timed out>', 'BestDR8.SpecZStatus': '<urlopen error timed out>', 'BestDR8.PhotoTag': '<urlopen error timed out>', 'BestDR8.Columns': '<urlopen error timed out>', 'BestDR8.SpecLineNames': '<urlopen error timed out>', 'BestDR8.SiteDBs': '<urlopen error timed out>', 'BestDR8.sdssPolygons': '<urlopen error timed out>', 'BestDR8.Star': '<urlopen error timed out>', 'BestDR8.QueryResults': '<urlopen error timed out>', 'BestDR8.SpecZWarning': '<urlopen error timed out>', 'BestDR8."FIRST"': '<urlopen error timed out>', 'BestDR8.RegionConvex': '<urlopen error timed out>', 'BestDR8.MaskType': '<urlopen error timed out>', 'BestDR8.segueSpecObjAll': '<urlopen error timed out>', 'BestDR8.sppLines': '<urlopen error timed out>', 'BestDR8."Zone"': '<urlopen error timed out>', 'BestDR8.ImageMask': '<urlopen error timed out>', 'BestDR8.ProgramType': '<urlopen error timed out>', 'BestDR8.SpecClass': '<urlopen error timed out>', 'BestDR8.PhotozRF': '<urlopen error timed out>', 'BestDR8.segue2SpecObjAll': '<urlopen error timed out>', 'BestDR8.sppParams': '<urlopen error timed out>', 'BestDR8.PhotoObjAll': '<urlopen error timed out>', 'BestDR8.RC3': '<urlopen error timed out>', 'BestDR8.PartitionMap': '<urlopen error timed out>', 'BestDR8.zooNoSpec': '<urlopen error timed out>', 'BestDR8.HalfSpace': '<urlopen error timed out>', 'BestDR8.PlateX': '<urlopen error timed out>', 'BestDR8.PhotoObjDR7': '<urlopen error timed out>', 'BestDR8.sdssTilingInfo': '<urlopen error timed out>', 'BestDR8.SpecPhoto': '<urlopen error timed out>', 'BestDR8.Unknown': '<urlopen error timed out>', 'BestDR8.zooMonochromeBias': '<urlopen error timed out>', 'BestDR8.PhotoFamily': '<urlopen error timed out>', 'BestDR8.Dependency': '<urlopen error timed out>'}
   1.141 +Table query completed after 10.012227535247803 seconds
   1.142 +
   1.143 +
   1.144 +
   1.145 +
   1.146 +## Restart Araybwyn VM, as it is not responding, and cannot be ssh'd into
   1.147 +
   1.148 +ssh trop02
   1.149 +
   1.150 +     ## Destroy VM
   1.151 +    vmname=Araybwyn
   1.152 +
   1.153 +    source "${HOME}/libvirt.settings"
   1.154 +    virsh \
   1.155 +        --connect "${connection}" \
   1.156 +         destroy \
   1.157 +            "${vmname:?}"
   1.158 +
   1.159 +   ## Startup VM
   1.160 +    vmname=Araybwyn
   1.161 +
   1.162 +    source "${HOME}/libvirt.settings"
   1.163 +    virsh \
   1.164 +        --connect "${connection}" \
   1.165 +         start \
   1.166 +            "${vmname:?}"
   1.167 +
   1.168 +
   1.169 +    ## Restart using compose
   1.170 +
   1.171 +	export buildtag=2.1.33
   1.172 +	source "${HOME:?}/chain.properties"
   1.173 +
   1.174 +	docker-compose --file "docker-compose.yml" run firethorn-py
   1.175 +
   1.176 +	Starting stevedore_jarmila_1  ... done
   1.177 +	Starting stevedore_bethany_1  ... done
   1.178 +	Starting stevedore_carolina_1 ... done
   1.179 +	Starting stevedore_gillian_1  ... done
   1.180 +	Python 3.5.2 (default, Nov 12 2018, 13:43:14) 
   1.181 +	[GCC 5.4.0 20160609] on linux
   1.182 +	Type "help", "copyright", "credits" or "license" for more information.
   1.183 +	>>> exit()
   1.184 +	[Stevedore@Araybwyn ~]$ docker ps
   1.185 +	CONTAINER ID        IMAGE                        COMMAND                  CREATED             STATUS                            PORTS                    NAMES
   1.186 +	44aaf5d76a44        firethorn/firethorn:2.1.33   "/bin/sh -c '/var/lo…"   6 days ago          Up 6 seconds (health: starting)   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   1.187 +	4ae7d97e1d09        firethorn/postgres:2.1.33    "docker-entrypoint.s…"   6 days ago          Up 38 seconds                     5432/tcp                 stevedore_carolina_1
   1.188 +	9099fe97de6d        firethorn/postgres:2.1.33    "docker-entrypoint.s…"   6 days ago          Up 38 seconds                     5432/tcp                 stevedore_bethany_1
   1.189 +	b75f992b86e7        firethorn/ogsadai:2.1.33     "/bin/sh -c '/var/lo…"   6 days ago          Up 38 seconds (healthy)           8080/tcp                 stevedore_jarmila_1
   1.190 +
   1.191 +
   1.192 +    ## curl http://tap.roe.ac.uk/firethorn/system/info still times out
   1.193 +    ## Looks like the Apache proxy service is not responding
   1.194 +    ## ssh   Stevedore@Acilamwen 
   1.195 +    ## docker restart apache
   1.196 +    ##  curl http://tap.roe.ac.uk/firethorn/system/info now works
   1.197 +
   1.198 +
   1.199 +
   1.200 +   ## Try a sync query:
   1.201 +   ## curl http://tap.roe.ac.uk/ssa/sync?REQUEST=doQuery&QUERY=SELECT+TOP+1+*+from+BestDR1.Sector&LANG=ADQL 
   1.202 +
   1.203 +
   1.204 +      <VOTABLE xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/v1.3" version="1.3">
   1.205 +<RESOURCE type="results">
   1.206 +<INFO name="QUERY_STATUS" value="OK"/>
   1.207 +<INFO name="link" value="http://tap.roe.ac.uk/firethorn/adql/table/29635605"/>
   1.208 +<TABLE ID="table.29635605" name="XX_YEXCG642EFIPCAAAAFVY6KMXB4">
   1.209 +<FIELD ID="column.29635659" name="regionID" datatype="int" arraysize="1" xtype="null">
   1.210 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635659"/>
   1.211 +</FIELD>
   1.212 +<FIELD ID="column.29635661" name="nTiles" datatype="int" arraysize="1" xtype="null">
   1.213 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635661"/>
   1.214 +</FIELD>
   1.215 +<FIELD ID="column.29635663" name="area" datatype="float" xtype="null">
   1.216 +<LINK content-type="application/json" content-role="metadata" href="http://tap.roe.ac.uk/firethorn/adql/column/29635663"/>
   1.217 +</FIELD>
   1.218 +<DATA>
   1.219 +<TABLEDATA>
   1.220 +<TR>
   1.221 +<TD>26816</TD>
   1.222 +<TD>1</TD>
   1.223 +<TD>0.0</TD>
   1.224 +</TR>
   1.225 +</TABLEDATA>
   1.226 +</DATA>
   1.227 +</TABLE>
   1.228 +</RESOURCE>
   1.229 +</VOTABLE>
   1.230 +
   1.231 +
   1.232 +   ## Service has recovered
   1.233 +
   1.234 +
   1.235 +