firethorn

changeset 4268:9fd44371154b 2.1.33-stv-tap-swarm

Notes on Deployment & Testing of Firethorn with Swarm
author Stelios <stv@roe.ac.uk>
date Fri Jul 19 17:40:19 2019 +0300 (2019-07-19)
parents e0f65ee972b9
children b3669dab2a60 40048e25928b
files doc/notes/stv/20190718-Firethorn-Swarm.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190718-Firethorn-Swarm.txt	Fri Jul 19 17:40:19 2019 +0300
     1.3 @@ -0,0 +1,536 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#  
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +
    1.27 +## ------------------------------------------------------------------------------------------
    1.28 +## Create Master VM 
    1.29 +## ------------------------------------------------------------------------------------------
    1.30 +
    1.31 +
    1.32 +createvm Froeseth (master)
    1.33 +..
    1.34 +Domain Froeseth started
    1.35 +
    1.36 +ssh Stevedore@Froeseth
    1.37 +
    1.38 +
    1.39 +
    1.40 +## ------------------------------------------------------------------------------------------
    1.41 +## Initialize Swarm with VM's address
    1.42 +## ------------------------------------------------------------------------------------------
    1.43 +
    1.44 +[Stevedore@Froeseth ~]$ ifconfig
    1.45 +
    1.46 +       ..  192.168.201.12  ..
    1.47 +
    1.48 +
    1.49 +docker swarm init --advertise-addr 192.168.201.12 
    1.50 +
    1.51 +
    1.52 +		Swarm initialized: current node (5zrs2po0x1teeyzigf09mw5vi) is now a manager.
    1.53 +
    1.54 +		To add a worker to this swarm, run the following command:
    1.55 +
    1.56 +		    docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
    1.57 +
    1.58 +		To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    1.59 +
    1.60 +
    1.61 +## ------------------------------------------------------------------------------------------
    1.62 +## Create Spark-net network
    1.63 +## ------------------------------------------------------------------------------------------
    1.64 +## docker network create --driver overlay firethorn-net
    1.65 +
    1.66 +
    1.67 +    ## Fix Firewall issue
    1.68 +    sudo su
    1.69 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.70 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.71 +	    firewall-cmd --add-port=7946/tcp --permanent
    1.72 +	    firewall-cmd --add-port=4789/tcp --permanent
    1.73 +	    firewall-cmd --add-port=4789/udp --permanent
    1.74 +	    iptables -A INPUT -p 50 -j ACCEPT    
    1.75 +	    firewall-cmd --reload
    1.76 +    exit
    1.77 +
    1.78 +
    1.79 +## -------------------------------------------------------------------------------------------
    1.80 +## Add secret function.
    1.81 +## -------------------------------------------------------------------------------------------
    1.82 +
    1.83 +    secrethost='stv@shepseskaf.roe.ac.uk'
    1.84 +    secretfile='${HOME:?}/secret.store...'
    1.85 +
    1.86 +    secret()
    1.87 +    {
    1.88 +        local key=${1:?}
    1.89 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    1.90 +    }
    1.91 +
    1.92 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    1.93 +
    1.94 +
    1.95 +
    1.96 +## -------------------------------------------------------------------------------------------
    1.97 +## Set the target branch and builtag
    1.98 +## -------------------------------------------------------------------------------------------
    1.99 +
   1.100 +
   1.101 +    export buildtag=2.1.33-stv-tap-swarm
   1.102 +    export branch=2.1.33-stv-tap-swarm
   1.103 +
   1.104 +
   1.105 +
   1.106 +## -------------------------------------------------------------------------------------------
   1.107 +## Create our chain.properties file
   1.108 +## -------------------------------------------------------------------------------------------
   1.109 +
   1.110 +cat > "${HOME:?}/chain.properties" << EOF
   1.111 +
   1.112 +metadata=postgres
   1.113 +metauser=$(pwgen 20 1)
   1.114 +metapass=$(pwgen 20 1) 
   1.115 +
   1.116 +userhost=$(secret 'firethorn.user.host')
   1.117 +userdata=$(secret 'firethorn.user.data')
   1.118 +useruser=$(secret 'firethorn.user.user')
   1.119 +userpass=$(secret 'firethorn.user.pass')
   1.120 +usertype=mssql
   1.121 +
   1.122 +datahost=$(secret 'firethorn.data.host')
   1.123 +datadata=$(secret 'firethorn.data.data')
   1.124 +datacatalog=$(secret 'firethorn.data.catalog')
   1.125 +datatype=$(secret 'firethorn.data.type')
   1.126 +datauser=$(secret 'firethorn.data.user')
   1.127 +datapass=$(secret 'firethorn.data.pass')
   1.128 +
   1.129 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   1.130 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   1.131 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   1.132 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   1.133 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   1.134 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   1.135 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   1.136 +
   1.137 +tunneluser=$(secret 'ssh.tunnel.user')
   1.138 +tunnelhost=$(secret 'ssh.tunnel.host')
   1.139 +
   1.140 +admingroup=$(pwgen 20 1)	
   1.141 +adminuser=$(pwgen 20 1)
   1.142 +adminpass=$(pwgen 20 1)
   1.143 +
   1.144 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   1.145 + 
   1.146 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   1.147 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   1.148 +
   1.149 +guestgroup=friends
   1.150 +endpoint=$(secret 'endpoint')
   1.151 +
   1.152 +firethornhost=gillian
   1.153 +
   1.154 +EOF
   1.155 +
   1.156 +
   1.157 +
   1.158 +## -----------------------------------------------------
   1.159 +## Create our Firethorn properties.
   1.160 +## -----------------------------------------------------
   1.161 +
   1.162 +source "${HOME:?}/chain.properties"
   1.163 +cat > "${HOME:?}/firethorn.properties" << EOF
   1.164 +
   1.165 +firethorn.meta.type=pgsql
   1.166 +firethorn.meta.driver=org.postgresql.Driver
   1.167 +firethorn.meta.host=bethany
   1.168 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   1.169 +firethorn.meta.data=postgres
   1.170 +firethorn.meta.user=${metauser:?}
   1.171 +firethorn.meta.pass=${metapass:?}
   1.172 +
   1.173 +firethorn.user.type=mssql
   1.174 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.175 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.176 +firethorn.user.host=${userhost:?}
   1.177 +firethorn.user.data=${userdata:?}
   1.178 +firethorn.user.user=${useruser:?}
   1.179 +firethorn.user.pass=${userpass:?}
   1.180 +
   1.181 +firethorn.tapschema.resource.name=${tapschemajdbc}
   1.182 +firethorn.tapschema.database.name=${tapschemadata}
   1.183 +firethorn.tapschema.database.host=${tapschemahost}
   1.184 +firethorn.tapschema.database.port=${tapschemaport}
   1.185 +firethorn.tapschema.database.user=${tapschemauser}
   1.186 +firethorn.tapschema.database.pass=${tapschemapass}
   1.187 +
   1.188 +firethorn.admin.community=${admingroup:?}
   1.189 +firethorn.admin.user.name=${adminuser:?}
   1.190 +firethorn.admin.user.pass=${adminpass:?}
   1.191 +
   1.192 +firethorn.guest.community=${guestgroup:?}
   1.193 +
   1.194 +adql.query.delay.first=${adql_query_delay_first:?}
   1.195 +
   1.196 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.197 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.198 +	
   1.199 +
   1.200 +EOF
   1.201 +
   1.202 +
   1.203 +
   1.204 +## -------------------------------------------------------------------------------------------
   1.205 +## Create our compose env files.
   1.206 +## -------------------------------------------------------------------------------------------
   1.207 +
   1.208 +source "${HOME:?}/chain.properties"
   1.209 +cat > "${HOME:?}/.env" << EOF
   1.210 +
   1.211 +buildtag=${branch:?}
   1.212 +endpoint=${endpoint:?}
   1.213 +
   1.214 +
   1.215 +metadata=postgres
   1.216 +metauser=${metauser:?}
   1.217 +metapass=${metapass:?}
   1.218 +firethorn.meta.type=pgsql
   1.219 +firethorn.meta.driver=org.postgresql.Driver
   1.220 +firethorn.meta.host=bethany
   1.221 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   1.222 +firethorn.meta.data=postgres
   1.223 +firethorn.meta.user=${metauser:?}
   1.224 +firethorn.meta.pass=${metapass:?}
   1.225 +
   1.226 +
   1.227 +datadata=${datadata:?}
   1.228 +datauser=${datauser:?}
   1.229 +datapass=${datapass:?}
   1.230 +datahost=${datahost:?}
   1.231 +datacatalog=${datacatalog:?}
   1.232 +datatype=mssql
   1.233 +
   1.234 +
   1.235 +tapschemadata=${tapschemadata:?}
   1.236 +tapschemauser=${tapschemauser:?}
   1.237 +tapschemapass=${tapschemapass:?}
   1.238 +tapschemahost=${tapschemahost}
   1.239 +tapschemaport=${tapschemaport:?}
   1.240 +tapschemajdbc=${tapschemajdbc:?}
   1.241 +tapschematype=${tapschematype:?}
   1.242 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   1.243 +firethorn.tapschema.database.name=${tapschemadata:?}
   1.244 +firethorn.tapschema.database.host=${tapschemahost:?}
   1.245 +firethorn.tapschema.database.port=5432
   1.246 +firethorn.tapschema.database.user=${tapschemauser:?}
   1.247 +firethorn.tapschema.database.pass=${tapschemapass:?}
   1.248 +firethorn.tapschema.database.type=pgsql
   1.249 +
   1.250 +userhost=${userhost:?}
   1.251 +userdata=${userdata:?}
   1.252 +useruser=${useruser:?}
   1.253 +userpass=${userpass:?}
   1.254 +usertype=mssql
   1.255 +firethorn.user.type=mssql
   1.256 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.257 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.258 +firethorn.user.host=${userhost:?}
   1.259 +firethorn.user.data=${userdata:?}
   1.260 +firethorn.user.user=${useruser:?}
   1.261 +firethorn.user.pass=${userpass:?}
   1.262 +
   1.263 +
   1.264 +admingroup=${admingroup:?}
   1.265 +adminuser=${adminuser:?}
   1.266 +adminpass=${adminpass:?}
   1.267 +guestgroup=${guestgroup:?}
   1.268 +firethorn.admin.community=${admingroup:?}
   1.269 +firethorn.admin.user.name=${adminuser:?}
   1.270 +firethorn.admin.user.pass=${adminpass:?}
   1.271 +firethorn.guest.community=${guestgroup:?}
   1.272 +
   1.273 +
   1.274 +adql_query_delay_first=${adql_query_delay_first:?}
   1.275 +adql.query.delay.first=${adql_query_delay_first:?}
   1.276 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   1.277 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   1.278 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.279 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.280 +
   1.281 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   1.282 +
   1.283 +firethornhost=gillian
   1.284 +
   1.285 +EOF
   1.286 +
   1.287 +
   1.288 +source "${HOME:?}/chain.properties"
   1.289 +cat > "${HOME:?}/.tapschema_env" << EOF
   1.290 +
   1.291 +POSTGRES_DB=${tapschemadatabase}
   1.292 +POSTGRES_USER=${tapschemauser}
   1.293 +POSTGRES_PASSWORD=${tapschemapass}
   1.294 +
   1.295 +EOF
   1.296 +
   1.297 +
   1.298 +source "${HOME:?}/chain.properties"
   1.299 +cat > "${HOME:?}/.metadata_env" << EOF
   1.300 +
   1.301 +POSTGRES_DB=${metadata}
   1.302 +POSTGRES_USER=${metauser}
   1.303 +POSTGRES_PASSWORD=${metapass}
   1.304 +         
   1.305 +EOF
   1.306 +
   1.307 +
   1.308 +## -------------------------------------------------------------------------------------------
   1.309 +## Fetch the TAP Deployer
   1.310 +## -------------------------------------------------------------------------------------------
   1.311 +
   1.312 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   1.313 +
   1.314 +
   1.315 +
   1.316 +## -------------------------------------------------------------------------------------------
   1.317 +## Build from Firethorn Source (2.1.33-stv-tap-swarm branch)
   1.318 +## -------------------------------------------------------------------------------------------
   1.319 +
   1.320 +sudo yum -y install maven
   1.321 +
   1.322 +hg clone http://wfau.metagrid.co.uk/code/firethorn hg
   1.323 +
   1.324 +
   1.325 +
   1.326 +cat > "${HOME:?}/firethorn.settings" << EOF
   1.327 +FIRETHORN_NAME=hg
   1.328 +FIRETHORN_BASE=/home/Stevedore/hg
   1.329 +FIRETHORN_CODE=/home/Stevedore/hg
   1.330 +FIRETHORN_CONF=/home/Stevedore/hg
   1.331 +EOF
   1.332 +
   1.333 +
   1.334 +
   1.335 +pushd ${HOME:?}/hg
   1.336 +
   1.337 +    hg update -C 2.1.33-stv-tap-swarm
   1.338 +
   1.339 +popd
   1.340 +
   1.341 +
   1.342 +
   1.343 +## -----------------------------------------------------
   1.344 +## Build our Docker images.
   1.345 +## -----------------------------------------------------
   1.346 +
   1.347 +    source "${HOME:?}/firethorn.settings"
   1.348 +    pushd "${FIRETHORN_CODE:?}"
   1.349 +
   1.350 +        export buildtag=$(hg branch)
   1.351 +
   1.352 +        docker-compose \
   1.353 +            --file docker/compose/images.yml \
   1.354 +            build
   1.355 +
   1.356 +    popd
   1.357 +
   1.358 +
   1.359 +
   1.360 +
   1.361 +##---------------------------------------------------------------------
   1.362 +## Compile our Java code.
   1.363 +##---------------------------------------------------------------------
   1.364 +
   1.365 +    source "${HOME:?}/firethorn.settings"
   1.366 +    pushd "${FIRETHORN_CODE:?}"
   1.367 +
   1.368 +        mvn clean install
   1.369 +
   1.370 +    popd
   1.371 +
   1.372 +
   1.373 +##---------------------------------------------------------------------
   1.374 +## Build our Java containers.
   1.375 +##---------------------------------------------------------------------
   1.376 +
   1.377 +    source "${HOME:?}/firethorn.settings"
   1.378 +    pushd "${FIRETHORN_CODE:?}"
   1.379 +
   1.380 +        export buildtag=$(hg branch)
   1.381 +        pushd firethorn-ogsadai/webapp
   1.382 +            mvn docker:package
   1.383 +        popd
   1.384 +
   1.385 +        export buildtag=$(hg branch)
   1.386 +        pushd firethorn-webapp
   1.387 +            mvn docker:package
   1.388 +        popd
   1.389 +
   1.390 +    popd
   1.391 +
   1.392 +
   1.393 +
   1.394 +## ------------------------------------------------------------------------
   1.395 +## Deploy Swarm stack
   1.396 +## ------------------------------------------------------------------------
   1.397 +
   1.398 +docker stack deploy -c docker-compose.yml ft
   1.399 +        .. 
   1.400 +	Creating service ft_bethany
   1.401 +	Creating service ft_carolina
   1.402 +	Creating service ft_jarmila
   1.403 +	Creating service ft_gillian
   1.404 +	Creating service ft_firethorn-py
   1.405 +     
   1.406 +
   1.407 +
   1.408 +[Stevedore@Froeseth ~]$ docker service ls
   1.409 +ID                  NAME                MODE                REPLICAS            IMAGE                                         PORTS
   1.410 +su0iy3uspkya        ft_bethany          replicated          1/1                 firethorn/postgres:2.1.33-stv-tap-swarm       
   1.411 +rve9iiumjoj4        ft_carolina         replicated          1/1                 firethorn/postgres:2.1.33-stv-tap-swarm       
   1.412 +7fswcw9btvct        ft_firethorn-py     replicated          0/1                 firethorn/firethorn-py:2.1.33-stv-tap-swarm   
   1.413 +l52nhtsaj5qy        ft_gillian          replicated          1/1                 firethorn/firethorn:2.1.33-stv-tap-swarm      *:8080->8080/tcp
   1.414 +2ubzla3luwe9        ft_jarmila          replicated          1/1                 firethorn/ogsadai:2.1.33-stv-tap-swarm 
   1.415 +
   1.416 +## ------------------------------------------------------------------------
   1.417 +## To remove all docker services 
   1.418 +## ------------------------------------------------------------------------
   1.419 +
   1.420 +## docker service rm $(docker service ls -q)
   1.421 +
   1.422 +
   1.423 +
   1.424 +## ------------------------------------------------------------------------
   1.425 +## Create TAP service with firethorn-utils
   1.426 +## ------------------------------------------------------------------------
   1.427 +
   1.428 +docker start ft_firethorn-py.1.txmiebaz5mohrlqo632ngb0x6
   1.429 +docker exec -it ft_firethorn-py.1.txmiebaz5mohrlqo632ngb0x6 bash
   1.430 +
   1.431 +root@firethorn-py:/home# python3
   1.432 +
   1.433 +
   1.434 +import firethorn
   1.435 +import firethorn_utils.configurator as configur
   1.436 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   1.437 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   1.438 +configurator =  configur.Configurator(ft)
   1.439 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   1.440 +
   1.441 +Importing JDBC Schema: ATLASDR1
   1.442 +Importing JDBC Schema: ATLASDR2
   1.443 +Importing JDBC Schema: ATLASDR3
   1.444 +Importing JDBC Schema: 2MASS
   1.445 +Importing JDBC Schema: 2XMM
   1.446 +Importing JDBC Schema: WISE
   1.447 +Importing JDBC Schema: VIKINGDR3
   1.448 +Importing JDBC Schema: VHSDR1
   1.449 +Importing JDBC Schema: MGC
   1.450 +Importing JDBC Schema: GLIMPSE
   1.451 +Importing JDBC Schema: FIRST
   1.452 +Importing JDBC Schema: DENIS
   1.453 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   1.454 +
   1.455 +
   1.456 +
   1.457 +	## ------------------------------------------------------------------------
   1.458 +## Update Apache Proxy 
   1.459 +## ------------------------------------------------------------------------
   1.460 +
   1.461 +[Stevedore@Gworewia ~]$ docker exec -it apache bash
   1.462 +
   1.463 +nano /etc/httpd/conf.d/tap.metagrid.xyz.conf
   1.464 +
   1.465 +    ProxyPassMatch    ^/osa\/(.*)$  http://froeseth:8080/firethorn/tap/54/$1
   1.466 +    ProxyPassReverse  ^/osa\/(.*)$  http://froeseth:8080/firethorn/tap/54/$1
   1.467 +
   1.468 +[Stevedore@Gworewia ~]$ docker kill --signal 'USR1' 'apache'
   1.469 +
   1.470 +
   1.471 +
   1.472 +## ------------------------------------------------------------------------
   1.473 +## Setup Local tunnel to Gworewia (Proxy)
   1.474 +## ------------------------------------------------------------------------
   1.475 +
   1.476 +## From local machine
   1.477 +ssh -L '*:8081:Gworewia:80' Stevedore@Gworewia 
   1.478 +
   1.479 +
   1.480 +
   1.481 +
   1.482 +## From local machine
   1.483 +
   1.484 +
   1.485 +cat parallel.py
   1.486 +
   1.487 +
   1.488 +import os                                                                       
   1.489 +from multiprocessing import Pool                                                
   1.490 +
   1.491 +process_list = range(100)
   1.492 +	
   1.493 +def run_process(process_counter):                                                             
   1.494 +    print ("Starting RUN # " + str(process_counter))
   1.495 +    os.system('python3.5 -c "import firethorn_utils.tap_validator as validator;validator.main()" -ft=http://localhost:8081/firethorn -r=54 -u= -p= -g= -v=True -m=sync')
   1.496 +    print ("Ended RUN # " + str(process_counter))
   1.497 +
   1.498 +
   1.499 +pool = Pool(processes=20)                                                        
   1.500 +pool.map(run_process, process_list)   
   1.501 +
   1.502 +
   1.503 +
   1.504 +.. 
   1.505 +
   1.506 +
   1.507 +
   1.508 +python3 parallel.py
   1.509 +
   1.510 +
   1.511 +..
   1.512 +
   1.513 +## After running this for about an hour, queries start failing, and firethorn stops responding
   1.514 +
   1.515 +[Stevedore@Froeseth ~]$ docker ps
   1.516 +CONTAINER ID        IMAGE                                      COMMAND                  CREATED             STATUS                             PORTS               NAMES
   1.517 +7619b2240e8c        firethorn/ogsadai:2.1.33-stv-tap-swarm     "/bin/sh -c '/var/lo…"   52 seconds ago      Up 47 seconds (health: starting)   8080/tcp            ft_jarmila.1.wlbqvl76nso5rqfhjv3bz45sj
   1.518 +be6c8a6c033f        firethorn/firethorn:2.1.33-stv-tap-swarm   "/bin/sh -c '/var/lo…"   4 minutes ago       Up 3 minutes (unhealthy)           8080/tcp            ft_gillian.1.agyapxmtenrr8harqboqheu7c
   1.519 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours                        5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
   1.520 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours                        5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
   1.521 +59b0ce098117        firethorn/firethorn-py:latest              "python3"                23 hours ago        Up 23 hours                                            ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
   1.522 +
   1.523 +
   1.524 +
   1.525 +
   1.526 +## Healthchecker is triggered and both Firethorn and Ogsadai recover with a "healthy" status
   1.527 +
   1.528 +
   1.529 +[Stevedore@Froeseth ~]$ docker ps
   1.530 +CONTAINER ID        IMAGE                                      COMMAND                  CREATED             STATUS                   PORTS               NAMES
   1.531 +5406b10d7fd2        firethorn/firethorn:2.1.33-stv-tap-swarm   "/bin/sh -c '/var/lo…"   3 minutes ago       Up 3 minutes (healthy)   8080/tcp            ft_gillian.1.zfo4zd8q2kgisk49d73xki4qj
   1.532 +7619b2240e8c        firethorn/ogsadai:2.1.33-stv-tap-swarm     "/bin/sh -c '/var/lo…"   4 minutes ago       Up 4 minutes (healthy)   8080/tcp            ft_jarmila.1.wlbqvl76nso5rqfhjv3bz45sj
   1.533 +698232c0b83f        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours              5432/tcp            ft_carolina.1.zfi7dbj2wn5v14ew8dq02xr4c
   1.534 +54d97f68dda2        firethorn/postgres:2.1.33-stv-tap-swarm    "docker-entrypoint.s…"   23 hours ago        Up 23 hours              5432/tcp            ft_bethany.1.4gdjm36l81crue5xykbrrm4gt
   1.535 +59b0ce098117        firethorn/firethorn-py:latest              "python3"                23 hours ago        Up 23 hours                                  ft_firethorn-py.1.nhjlot9nvyoitf5onbde0l8gj
   1.536 +
   1.537 +
   1.538 +
   1.539 +## Queries start working again..