firethorn

changeset 4208:b511bd7b05d7 2.1.28-tap-issues

Notes on Failed VM
author Stelios <stv@roe.ac.uk>
date Fri Apr 26 14:55:15 2019 +0300 (2019-04-26)
parents 27c53ddd01e6
children 68d026ea0895
files doc/notes/stv/20190425-OSA-TAP-Debugging.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190425-OSA-TAP-Debugging.txt	Fri Apr 26 14:55:15 2019 +0300
     1.3 @@ -0,0 +1,444 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +
    1.27 +ssh Stevedore@Araybwyn
    1.28 +ssh: connect to host araybwyn port 22: No route to host
    1.29 +
    1.30 +
    1.31 +# -----------------------------------------------------
    1.32 +# List the active virtual machines.
    1.33 +# stv@trop02:~$
    1.34 +	
    1.35 +	virsh \
    1.36 +	>         --connect "${connection}" \
    1.37 +	>         list
    1.38 +
    1.39 +	setlocale: No such file or directory
    1.40 + 
    1.41 +Id    Name                           State
    1.42 +----------------------------------------------------
    1.43 + 3     Acilamwen                      running
    1.44 + 6     Araybwyn                       running
    1.45 +
    1.46 +
    1.47 +
    1.48 +
    1.49 +# -----------------------------------------------------
    1.50 +# Check disk space of all VMs
    1.51 +
    1.52 +
    1.53 +vmlist=(Acilamwen Araybwyn)
    1.54 +
    1.55 +for vmname in ${vmlist[*]}
    1.56 +>     do
    1.57 +>         echo "--"
    1.58 +>         echo "Name [${vmname}]"
    1.59 +>         ssh "${vmname:?}" \
    1.60 +>             "
    1.61 +>             df -h
    1.62 +>             "
    1.63 +>     done
    1.64 +--
    1.65 +Name [Acilamwen]
    1.66 +Permission denied (publickey,gssapi-keyex,gssapi-with-mic).
    1.67 +--
    1.68 +Name [Araybwyn]
    1.69 +ssh: connect to host araybwyn port 22: No route to host
    1.70 +
    1.71 +
    1.72 +
    1.73 +
    1.74 +# -----------------------------------------------------
    1.75 +# Reboot the firethorn VM.
    1.76 +# stv@trop02:~$
    1.77 +
    1.78 +    vmname=Araybwyn
    1.79 +    source "${HOME}/libvirt.settings"
    1.80 +
    1.81 +	virsh \
    1.82 +	--connect "${connection}" \
    1.83 +	reboot \
    1.84 +	    "${vmname:?}"
    1.85 +
    1.86 +	setlocale: No such file or directory
    1.87 +	Domain Araybwyn is being rebooted
    1.88 +
    1.89 +
    1.90 +# -----------------------------------------------------
    1.91 +# Login to the firethorn VM.
    1.92 +# stv@trop02:~$
    1.93 +
    1.94 +    vmname=Araybwyn
    1.95 +
    1.96 +    ssh "${vmname:?}"
    1.97 +    ssh: connect to host araybwyn port 22: No route to host
    1.98 +
    1.99 +
   1.100 +# -----------------------------------------------------
   1.101 +# Check the VM status.
   1.102 +# stv@trop02:~$
   1.103 +
   1.104 +    vmname=Araybwyn
   1.105 +
   1.106 +    source "${HOME}/libvirt.settings"
   1.107 +    virsh \
   1.108 +        --connect "${connection}" \
   1.109 +        dominfo \
   1.110 +            "${vmname:?}"
   1.111 +
   1.112 +Id:             6
   1.113 +Name:           Araybwyn
   1.114 +UUID:           a65ad4ca-0662-4c43-a32c-91e4180d87bc
   1.115 +OS Type:        hvm
   1.116 +State:          running
   1.117 +CPU(s):         4
   1.118 +CPU time:       783601.4s
   1.119 +Max memory:     4194304 KiB
   1.120 +Used memory:    4194304 KiB
   1.121 +Persistent:     yes
   1.122 +Autostart:      disable
   1.123 +Managed save:   no
   1.124 +Security model: none
   1.125 +Security DOI:   0
   1.126 +
   1.127 +
   1.128 +
   1.129 +    virsh \
   1.130 +        --connect "${connection}" \
   1.131 +        domstate \
   1.132 +            "${vmname:?}"
   1.133 +	
   1.134 +	running
   1.135 +
   1.136 +
   1.137 +
   1.138 +
   1.139 +    virsh \
   1.140 +        --connect "${connection}" \
   1.141 +        domstats \
   1.142 +            "${vmname:?}"
   1.143 +
   1.144 +	setlocale: No such file or directory
   1.145 +	Domain: 'Araybwyn'
   1.146 +	  state.state=1
   1.147 +	  state.reason=1
   1.148 +	  cpu.time=785415372081224
   1.149 +	  cpu.user=36850540000000
   1.150 +	  cpu.system=137433620000000
   1.151 +	  balloon.current=4194304
   1.152 +	  balloon.maximum=4194304
   1.153 +	  vcpu.current=4
   1.154 +	  vcpu.maximum=4
   1.155 +	  vcpu.0.state=1
   1.156 +	  vcpu.0.time=191511150000000
   1.157 +	  vcpu.1.state=1
   1.158 +	  vcpu.1.time=160686720000000
   1.159 +	  vcpu.2.state=1
   1.160 +	  vcpu.2.time=146815730000000
   1.161 +	  vcpu.3.state=1
   1.162 +	  vcpu.3.time=133039200000000
   1.163 +	  net.count=1
   1.164 +	  net.0.name=vnet0
   1.165 +	  net.0.rx.bytes=8760633485
   1.166 +	  net.0.rx.pkts=40815593
   1.167 +	  net.0.rx.errs=0
   1.168 +	  net.0.rx.drop=0
   1.169 +	  net.0.tx.bytes=17038380088
   1.170 +	  net.0.tx.pkts=39162201
   1.171 +	  net.0.tx.errs=0
   1.172 +	  net.0.tx.drop=0
   1.173 +	  block.count=2
   1.174 +	  block.0.name=vda
   1.175 +	  block.0.rd.reqs=4681231
   1.176 +	  block.0.rd.bytes=101605817344
   1.177 +	  block.0.rd.times=2968455437655
   1.178 +	  block.0.wr.reqs=520676112
   1.179 +	  block.0.wr.bytes=47246008038400
   1.180 +	  block.0.wr.times=273501681954057
   1.181 +	  block.0.fl.reqs=58771451
   1.182 +	  block.0.fl.times=128714065178477
   1.183 +	  block.1.name=vdb
   1.184 +	  block.1.rd.reqs=48
   1.185 +	  block.1.rd.bytes=186368
   1.186 +	  block.1.rd.times=2764623
   1.187 +	  block.1.wr.reqs=0
   1.188 +	  block.1.wr.bytes=0
   1.189 +	  block.1.wr.times=0
   1.190 +	  block.1.fl.reqs=0
   1.191 +	  block.1.fl.times=0
   1.192 +
   1.193 +
   1.194 +
   1.195 +
   1.196 +	virsh \
   1.197 +	 --connect "${connection}" \
   1.198 +	 dommemstat \
   1.199 +	     "${vmname:?}"
   1.200 +
   1.201 +
   1.202 +	actual 4194304
   1.203 +	swap_in 0
   1.204 +	swap_out 0
   1.205 +	major_fault 106
   1.206 +	minor_fault 65006
   1.207 +	unused 3792832
   1.208 +	available 4038224
   1.209 +	rss 4321284
   1.210 +
   1.211 +
   1.212 +
   1.213 +
   1.214 +    virsh \
   1.215 +        --connect "${connection}" \
   1.216 +        domblklist \
   1.217 +            "${vmname:?}"
   1.218 +
   1.219 +	setlocale: No such file or directory
   1.220 +	Target     Source
   1.221 +	------------------------------------------------
   1.222 +	vda        /libvirt/storage/live/Araybwyn.qcow
   1.223 +	vdb        /var/lib/libvirt/images/init/Araybwyn.iso
   1.224 +
   1.225 +
   1.226 +	
   1.227 +
   1.228 +    virsh \
   1.229 +        --connect "${connection}" \
   1.230 +        domblkinfo \
   1.231 +            "${vmname:?}" \
   1.232 +            'vda'
   1.233 +
   1.234 +	setlocale: No such file or directory
   1.235 +
   1.236 +	Capacity:       34359738368
   1.237 +	Allocation:     27541970944
   1.238 +	Physical:       27541970944
   1.239 +
   1.240 +
   1.241 +	
   1.242 +    virsh \
   1.243 +        --connect "${connection}" \
   1.244 +        domblkerror \
   1.245 +            "${vmname:?}"
   1.246 +
   1.247 +	setlocale: No such file or directory
   1.248 +	No errors found
   1.249 +
   1.250 +
   1.251 +
   1.252 +    virsh \
   1.253 +        --connect "${connection}" \
   1.254 +        domblkstat \
   1.255 +            "${vmname:?}" \
   1.256 +            'vda'
   1.257 +
   1.258 +	setlocale: No such file or directory
   1.259 +
   1.260 +	vda rd_req 4681231
   1.261 +	vda rd_bytes 101605817344
   1.262 +	vda wr_req 520676112
   1.263 +	vda wr_bytes 47246008038400
   1.264 +	vda flush_operations 58771451
   1.265 +	vda rd_total_times 2968455437655
   1.266 +	vda wr_total_times 273501681954057
   1.267 +	vda flush_total_times 128714065178477
   1.268 +
   1.269 +
   1.270 +
   1.271 +
   1.272 +#
   1.273 +# Reboot didn't reboot. 
   1.274 +#
   1.275 +
   1.276 +
   1.277 +    vmname=Araybwyn
   1.278 +
   1.279 +    source "${HOME}/libvirt.settings"
   1.280 +    virsh \
   1.281 +        --connect "${connection}" \
   1.282 +        reboot \
   1.283 +            "${vmname:?}"
   1.284 +
   1.285 +       Domain Araybwyn is being rebooted
   1.286 +
   1.287 +
   1.288 +    ssh Stevedore@Araybwyn 
   1.289 +    ssh: connect to host araybwyn port 22: No route to host
   1.290 +
   1.291 +
   1.292 +# -----------------------------------------------------
   1.293 +# Try pulling the power cord ...
   1.294 +# stv@trop02:~$
   1.295 +
   1.296 +    vmname=Araybwyn
   1.297 +
   1.298 +    source "${HOME}/libvirt.settings"
   1.299 +    virsh \
   1.300 +        --connect "${connection}" \
   1.301 +         destroy \
   1.302 +            "${vmname:?}"
   1.303 +
   1.304 +    setlocale: No such file or directory
   1.305 +    Domain Araybwyn destroyed
   1.306 +
   1.307 +
   1.308 +
   1.309 +# -----------------------------------------------------
   1.310 +# Try start it up again ...
   1.311 +# stv@trop02:~$
   1.312 +
   1.313 +    vmname=Araybwyn
   1.314 +
   1.315 +    source "${HOME}/libvirt.settings"
   1.316 +    virsh \
   1.317 +        --connect "${connection}" \
   1.318 +         start \
   1.319 +            "${vmname:?}"
   1.320 +
   1.321 +    setlocale: No such file or directory
   1.322 +    Domain Araybwyn started
   1.323 +
   1.324 +
   1.325 +# -----------------------------------------------------
   1.326 +# Login and check disk space ...
   1.327 +
   1.328 +stv@trop02:~$ ssh Stevedore@Araybwyn 
   1.329 +Warning: Permanently added 'araybwyn' (ECDSA) to the list of known hosts.
   1.330 +Last login: Wed Jan 30 18:18:46 2019 from 192.168.202.1
   1.331 +
   1.332 +	df -h
   1.333 +	Filesystem      Size  Used Avail Use% Mounted on
   1.334 +	devtmpfs        2.0G     0  2.0G   0% /dev
   1.335 +	tmpfs           2.0G     0  2.0G   0% /dev/shm
   1.336 +	tmpfs           2.0G  532K  2.0G   1% /run
   1.337 +	tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.338 +	/dev/vda3        31G   25G  5.5G  82% /
   1.339 +	tmpfs           2.0G  4.0K  2.0G   1% /tmp
   1.340 +	/dev/vda1       240M   89M  135M  40% /boot
   1.341 +	tmpfs           395M     0  395M   0% /run/user/1001
   1.342 +
   1.343 +
   1.344 +	docker ps -a
   1.345 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS                    NAMES
   1.346 +2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                5 weeks ago         Exited (255) 3 minutes ago                            stevedore_firethorn-py_run_4
   1.347 +659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                               stevedore_firethorn-py_run_3
   1.348 +cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                             stevedore_firethorn-py_run_2
   1.349 +ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                             stevedore_firethorn-py_run_1
   1.350 +513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 3 minutes ago   0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   1.351 +f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (255) 3 minutes ago   8080/tcp                 stevedore_jarmila_1
   1.352 +7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 3 minutes ago   5432/tcp                 stevedore_carolina_1
   1.353 +80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (255) 3 minutes ago   5432/tcp                 stevedore_bethany_1
   1.354 +
   1.355 +
   1.356 +
   1.357 +# -----------------------------------------------------
   1.358 +# Restart using compose
   1.359 +
   1.360 +	export buildtag=2.1.28
   1.361 +	source "${HOME:?}/chain.properties"
   1.362 +
   1.363 +	docker-compose --file "docker-compose.yml" run firethorn-py
   1.364 +
   1.365 +	Starting stevedore_bethany_1  ... done
   1.366 +	Starting stevedore_jarmila_1  ... done
   1.367 +	Starting stevedore_carolina_1 ... done
   1.368 +	Starting stevedore_gillian_1  ... done
   1.369 +
   1.370 +Python 3.5.2 (default, Nov 12 2018, 13:43:14) 
   1.371 +[GCC 5.4.0 20160609] on linux
   1.372 +Type "help", "copyright", "credits" or "license" for more information.
   1.373 +
   1.374 +>>> 
   1.375 +
   1.376 +import os
   1.377 +import uuid
   1.378 +import time
   1.379 +import firethorn as ftpy
   1.380 +
   1.381 +#
   1.382 +# Create our Firethorn client.
   1.383 +firethorn = ftpy.Firethorn(
   1.384 +    endpoint = os.environ.get(
   1.385 +        'endpoint'
   1.386 +        )
   1.387 +    )
   1.388 +
   1.389 +#
   1.390 +# Login using a guest account.
   1.391 +firethorn.login(
   1.392 +    str(uuid.uuid4()),
   1.393 +    str(uuid.uuid4()),
   1.394 +    None
   1.395 +    )
   1.396 +
   1.397 +# Get the Atlas AdqlResource.
   1.398 +resource =firethorn.firethorn_engine.select_adql_resource_by_ident("http://tap.roe.ac.uk/firethorn/adql/resource/54")
   1.399 +
   1.400 +
   1.401 +#
   1.402 +# Create and run a query on the resource.
   1.403 +query_str = "SELECT TOP 1000 ra, dec FROM ATLASDR1.atlasSource"
   1.404 +query_obj = resource.create_query(
   1.405 +    query_str,
   1.406 +    "COMPLETED",
   1.407 +    None,
   1.408 +    3000000
   1.409 +    )
   1.410 +
   1.411 +
   1.412 +
   1.413 +print(
   1.414 +    query_obj.table()
   1.415 +     )
   1.416 +{
   1.417 +  "modified": "2019-04-25T23:27:06.596",
   1.418 +  "created": "2019-04-25T23:27:06.596",
   1.419 +  "self": "http://tap.roe.ac.uk/firethorn/adql/table/56637553",
   1.420 +  "depth": "PARTIAL",
   1.421 +  "type": "http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json",
   1.422 +  "root": "http://tap.roe.ac.uk/firethorn/jdbc/table/56637552",
   1.423 +  "columns": "http://tap.roe.ac.uk/firethorn/adql/table/56637553/columns/select",
   1.424 +  "fullname": "temp.XX_HYK3D7TV7462UAAAAFVFNUXTME",
   1.425 +  "text": null,
   1.426 +  "base": "http://tap.roe.ac.uk/firethorn/jdbc/table/56637552",
   1.427 +  "query": "http://tap.roe.ac.uk/firethorn/blue/query/56637352",
   1.428 +  "parent": "http://tap.roe.ac.uk/firethorn/adql/schema/56637502",
   1.429 +  "ident": "56637553",
   1.430 +  "owner": "http://tap.roe.ac.uk/firethorn/community-member/56637258",
   1.431 +  "name": "XX_HYK3D7TV7462UAAAAFVFNUXTME",
   1.432 +  "url": "http://tap.roe.ac.uk/firethorn/adql/table/56637553",
   1.433 +  "resource": "http://tap.roe.ac.uk/firethorn/adql/resource/56637452",
   1.434 +  "formats": {
   1.435 +    "votable": "http://tap.roe.ac.uk/firethorn/adql/table/56637553/votable",
   1.436 +    "datatable": "http://tap.roe.ac.uk/firethorn/adql/table/56637553/datatable"
   1.437 +  },
   1.438 +  "metadata": {
   1.439 +    "adql": {
   1.440 +      "count": 1000,
   1.441 +      "status": "COMPLETED"
   1.442 +    }
   1.443 +  },
   1.444 +  "schema": "http://tap.roe.ac.uk/firethorn/adql/schema/56637502"
   1.445 +}
   1.446 +
   1.447 +