firethorn

changeset 4224:02a6b0a80e23

Increased the VM memory to 8G
author Dave Morris <dmr@roe.ac.uk>
date Fri May 10 10:31:08 2019 +0100 (2019-05-10)
parents b4d1a8a6ee46
children 1e7585468a10
files doc/notes/zrq/20190508-02-vmdown-debug.txt
line diff
     1.1 --- a/doc/notes/zrq/20190508-02-vmdown-debug.txt	Thu May 09 12:43:44 2019 +0100
     1.2 +++ b/doc/notes/zrq/20190508-02-vmdown-debug.txt	Fri May 10 10:31:08 2019 +0100
     1.3 @@ -109,3 +109,297 @@
     1.4  --END--
     1.5  
     1.6  
     1.7 +# -----------------------------------------------------
     1.8 +# Check the system status.
     1.9 +#[user@desktop]
    1.10 +
    1.11 +    http://tap.roe.ac.uk/firethorn/system/info
    1.12 +
    1.13 +    {
    1.14 +    ....
    1.15 +    "memory" : {
    1.16 +        "total" : 800063488,
    1.17 +        "free" : 271500784,
    1.18 +        "max" : 919076864
    1.19 +        },
    1.20 +    "disk" : {
    1.21 +        "total" : 33016512512,
    1.22 +        "free" : 9332752384,
    1.23 +        "usable" : 8260763648
    1.24 +        }
    1.25 +    ....
    1.26 +    "time" : "2019-05-09T13:22:43.488Z"
    1.27 +    ....
    1.28 +    }
    1.29 +
    1.30 +
    1.31 +# -----------------------------------------------------
    1.32 +# Check the system status.
    1.33 +#[user@desktop]
    1.34 +
    1.35 +    http://tap.roe.ac.uk/firethorn/system/info
    1.36 +
    1.37 +    {
    1.38 +    ....
    1.39 +    "memory" : {
    1.40 +        "total" : 788529152,
    1.41 +        "free" : 277734984,
    1.42 +        "max" : 919076864
    1.43 +        },
    1.44 +    "disk" : {
    1.45 +        "total" : 33016512512,
    1.46 +        "free" : 9326923776,
    1.47 +        "usable" : 8255049728
    1.48 +        }
    1.49 +    ....
    1.50 +    "time" : "2019-05-09T15:51:46.214Z",
    1.51 +    ....
    1.52 +    }
    1.53 +
    1.54 +    {
    1.55 +    ....
    1.56 +    "memory" : {
    1.57 +        "total" : 791674880,
    1.58 +        "free" : 306134816,
    1.59 +        "max" : 919076864
    1.60 +        },
    1.61 +    "disk" : {
    1.62 +        "total" : 33016512512,
    1.63 +        "free" : 9319641088,
    1.64 +        "usable" : 8247816192
    1.65 +        }
    1.66 +    ....
    1.67 +    "time" : "2019-05-09T17:23:06.860Z",
    1.68 +    ....
    1.69 +    }
    1.70 +
    1.71 +    {
    1.72 +    ....
    1.73 +    "memory" : {
    1.74 +        "total" : 791150592,
    1.75 +        "free" : 303769264,
    1.76 +        "max" : 919076864
    1.77 +        },
    1.78 +    "disk" : {
    1.79 +        "total" : 33016512512,
    1.80 +        "free" : 9320611840,
    1.81 +        "usable" : 8248885248
    1.82 +        }
    1.83 +    ....
    1.84 +    "time" : "2019-05-09T18:34:11.381Z",
    1.85 +    ....
    1.86 +    }
    1.87 +
    1.88 +
    1.89 +22:40 - fails
    1.90 +
    1.91 +
    1.92 +
    1.93 +# -----------------------------------------------------
    1.94 +# Check the system status.
    1.95 +#[user@virtual]
    1.96 +
    1.97 +    date
    1.98 +
    1.99 +--START--
   1.100 +Thu  9 May 22:45:35 BST 2019
   1.101 +--END--
   1.102 +
   1.103 +free
   1.104 +
   1.105 +--START--
   1.106 +              total        used        free      shared  buff/cache   available
   1.107 +Mem:        4038224     1776084      220624      159980     2041516     1871952
   1.108 +Swap:       1048572       88320      960252
   1.109 +--END--
   1.110 +
   1.111 +free -h
   1.112 +
   1.113 +--START--
   1.114 +              total        used        free      shared  buff/cache   available
   1.115 +Mem:           3.9G        1.7G        172M        156M        2.0G        1.8G
   1.116 +Swap:          1.0G         86M        937M
   1.117 +--END--
   1.118 +
   1.119 +    df -h
   1.120 +
   1.121 +--START--
   1.122 +Filesystem      Size  Used Avail Use% Mounted on
   1.123 +devtmpfs        2.0G     0  2.0G   0% /dev
   1.124 +tmpfs           2.0G     0  2.0G   0% /dev/shm
   1.125 +tmpfs           2.0G  828K  2.0G   1% /run
   1.126 +tmpfs           2.0G     0  2.0G   0% /sys/fs/cgroup
   1.127 +/dev/vda3        31G   23G  7.7G  75% /
   1.128 +tmpfs           2.0G  4.0K  2.0G   1% /tmp
   1.129 +/dev/vda1       240M   89M  135M  40% /boot
   1.130 +tmpfs           395M     0  395M   0% /run/user/1001
   1.131 +--END--
   1.132 +
   1.133 +    #
   1.134 +    # Memory not an issue (1.8G available).
   1.135 +    # Disc space not an issue (7.7G available).
   1.136 +    #
   1.137 +
   1.138 +    docker ps -a
   1.139 +
   1.140 +--START--
   1.141 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                      PORTS                    NAMES
   1.142 +058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                18 hours ago        Exited (0) 17 hours ago                              stevedore_firethorn-py_run_7
   1.143 +e6deacef850b        firethorn/firethorn-py:2.1.28   "python3"                26 hours ago        Exited (137) 18 hours ago                            stevedore_firethorn-py_run_6
   1.144 +16d4c87bc6c3        firethorn/firethorn-py:2.1.28   "python3"                13 days ago         Exited (137) 13 days ago                             stevedore_firethorn-py_run_5
   1.145 +2fe430d3956d        firethorn/firethorn-py:2.1.28   "python3"                7 weeks ago         Exited (255) 13 days ago                             stevedore_firethorn-py_run_4
   1.146 +659eaacbdddc        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (0) 2 months ago                              stevedore_firethorn-py_run_3
   1.147 +cc174a20a0c7        firethorn/firethorn-py:2.1.28   "python3"                2 months ago        Exited (255) 2 months ago                            stevedore_firethorn-py_run_2
   1.148 +ebed74a70e56        firethorn/firethorn-py:2.1.28   "python3"                4 months ago        Exited (137) 4 months ago                            stevedore_firethorn-py_run_1
   1.149 +513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (unhealthy)     0.0.0.0:8080->8080/tcp   stevedore_gillian_1
   1.150 +f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Up 18 hours (healthy)       8080/tcp                 stevedore_jarmila_1
   1.151 +7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_carolina_1
   1.152 +80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Up 18 hours                 5432/tcp                 stevedore_bethany_1
   1.153 +--END--
   1.154 +
   1.155 +    #
   1.156 +    # Tomcat is unhealthy because it is not responding to HTTP requests for system status.
   1.157 +    # Vitriual machine is still healthy because it accepted my ssh login.
   1.158 +    #
   1.159 +
   1.160 +# -----------------------------------------------------
   1.161 +# Check the firethorn logs.
   1.162 +#[user@virtual]
   1.163 +
   1.164 +    docker exec -it stevedore_gillian_1 bash
   1.165 +
   1.166 +        tail -n 100 logs/firethorn-debug.log
   1.167 +
   1.168 +--START--
   1.169 +2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [BaseProtector] isAdmin(Identity) [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   1.170 +2019-05-09 21:48:00,017 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] admin()
   1.171 +2019-05-09 21:48:00,018 DEBUG [FireThornTaskScheduler-84] [IdentityEntity] find (Community, String) [Jo9keiRu7moo6ooNee1s]
   1.172 +2019-05-09 21:48:00,019 DEBUG [FireThornTaskScheduler-84] [IdentityEntity]   found [entity[class[uk.ac.roe.wfau.firethorn.identity.IdentityEntity] ident[163]]]
   1.173 +2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [UserDataCleaner]   schema [252][FirethornUserdataTAP20181212LIVE.dbo]
   1.174 +2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] tables() for [252][FirethornUserdataTAP20181212LIVE.dbo]
   1.175 +2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scan for [252][FirethornUserdataTAP20181212LIVE.dbo]
   1.176 +2019-05-09 21:48:00,022 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scantest for [252][FirethornUserdataTAP20181212LIVE.dbo]
   1.177 +....
   1.178 +....
   1.179 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcSchemaEntity] factory()
   1.180 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   1.181 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   1.182 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   1.183 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   1.184 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   1.185 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   1.186 +2019-05-09 21:50:00,026 DEBUG [FireThornTaskScheduler-84] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:50:00.017Z]
   1.187 +--END--
   1.188 +
   1.189 +
   1.190 +        tail -n 200 logs/firethorn-trace.log
   1.191 +
   1.192 +--START--
   1.193 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory] scanperiod()
   1.194 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity$EntityFactory]   value [PT25H]
   1.195 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] Factory scanperiod  [PT25H]
   1.196 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prevscan   [2019-05-09T20:28:00.081Z]
   1.197 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] scanperiod [PT25H]
   1.198 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseComponentEntity] prev scan is recent - skipping
   1.199 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   1.200 +2019-05-09 21:49:00,008 DEBUG [FireThornTaskScheduler-84] [BaseProtector] affirm(Authentications, Action) [uk.ac.roe.wfau.firethorn.access.BaseAction@1c69285]
   1.201 +....
   1.202 +....
   1.203 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory] scanperiod()
   1.204 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity$EntityFactory]   value [PT25H]
   1.205 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] Factory scanperiod  [PT25H]
   1.206 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prevscan   [2019-05-09T20:30:08.178Z]
   1.207 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] scanperiod [PT25H]
   1.208 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [BaseComponentEntity] prev scan is recent - skipping
   1.209 +2019-05-09 21:52:00,023 DEBUG [FireThornTaskScheduler-93] [JdbcTableEntity] pending(JdbcSchema, DateTime) [252][FirethornUserdataTAP20181212LIVE.dbo] [2019-05-08T21:52:00.014Z]
   1.210 +2019-05-09 21:52:00,194 TRACE [FireThornTaskScheduler-93] [UserDataCleaner] -- DONE --
   1.211 +
   1.212 +    #
   1.213 +    # Need to setup a ssh tunnel for the Java debugger to see if Threads are an issue.
   1.214 +    #
   1.215 +
   1.216 +    #
   1.217 +    # Reboot the VM ..
   1.218 +    #
   1.219 +
   1.220 +# -----------------------------------------------------
   1.221 +# Increase the virtual machine memory.
   1.222 +#[user@trop02]
   1.223 +
   1.224 +    vmname=Araybwyn
   1.225 +
   1.226 +    source ${HOME}/libvirt.settings
   1.227 +
   1.228 +    virsh -c ${connection:?} \
   1.229 +        dumpxml "${vmname}"
   1.230 +
   1.231 +--START--
   1.232 +<domain type='kvm' id='9'>
   1.233 +  <name>Araybwyn</name>
   1.234 +  <uuid>a65ad4ca-0662-4c43-a32c-91e4180d87bc</uuid>
   1.235 +  <memory unit='KiB'>4194304</memory>
   1.236 +  <currentMemory unit='KiB'>4194304</currentMemory>
   1.237 +  <vcpu placement='static'>4</vcpu>
   1.238 +  <resource>
   1.239 +    <partition>/machine</partition>
   1.240 +  </resource>
   1.241 +  ....
   1.242 +</domain>
   1.243 +--END--
   1.244 +
   1.245 +    virsh -c ${connection:?} \
   1.246 +        shutdown "${vmname}"
   1.247 +
   1.248 +    virsh -c ${connection:?} \
   1.249 +        setmaxmem "${vmname}" 8GB --config
   1.250 +
   1.251 +    virsh -c ${connection:?} \
   1.252 +        setmem    "${vmname}" 8GB --config
   1.253 +
   1.254 +    virsh -c ${connection:?} \
   1.255 +        start "${vmname}"
   1.256 +
   1.257 +# -----------------------------------------------------
   1.258 +# Login to the virtual machine.
   1.259 +#[user@trop02]
   1.260 +
   1.261 +    ssh "${vmname}"
   1.262 +
   1.263 +# -----------------------------------------------------
   1.264 +# Check the Docker containers.
   1.265 +#[user@virtual]
   1.266 +
   1.267 +    docker ps -a
   1.268 +
   1.269 +--START--
   1.270 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                       PORTS               NAMES
   1.271 +058615fbf37e        firethorn/firethorn-py:2.1.28   "python3"                19 hours ago        Exited (0) 18 hours ago                          stevedore_firethorn-py_run_7
   1.272 +....
   1.273 +513af3cb732e        firethorn/firethorn:2.1.28      "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_gillian_1
   1.274 +f057320941a1        firethorn/ogsadai:2.1.28        "/bin/sh -c '/var/lo…"   4 months ago        Exited (143) 2 minutes ago                       stevedore_jarmila_1
   1.275 +7ed501c87d8f        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_carolina_1
   1.276 +80da24dbaf6a        firethorn/postgres:2.1.28       "docker-entrypoint.s…"   4 months ago        Exited (0) 2 minutes ago                         stevedore_bethany_1
   1.277 +--END--
   1.278 +
   1.279 +# -----------------------------------------------------
   1.280 +# Restart our services..
   1.281 +#[user@virtual]
   1.282 +
   1.283 +    export buildtag=2.1.28
   1.284 +
   1.285 +    source "${HOME:?}/chain.properties"
   1.286 +    docker-compose --file "docker-compose.yml" run firethorn-py
   1.287 +
   1.288 +--START--
   1.289 +Starting stevedore_carolina_1 ... done
   1.290 +Starting stevedore_jarmila_1  ... done
   1.291 +Starting stevedore_bethany_1  ... done
   1.292 +Starting stevedore_gillian_1  ... done
   1.293 +Python 3.5.2 (default, Nov 12 2018, 13:43:14)
   1.294 +[GCC 5.4.0 20160609] on linux
   1.295 +Type "help", "copyright", "credits" or "license" for more information.
   1.296 +>>>
   1.297 +--END--
   1.298 +
   1.299 +
   1.300 +