1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190514-Firethorn-utils-Healthchecker-run.txt	Wed May 20 14:18:48 2020 +0300
     1.3 @@ -0,0 +1,65 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#  
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +#
    1.26 +
    1.27 +# ---------------------------------------------
    1.28 +# Fetch and install Firethorn Python Libraries
    1.29 +# ---------------------------------------------
    1.30 +
    1.31 +git clone https://github.com/stvoutsin/firethorn.py
    1.32 +git clone https://github.com/stvoutsin/firethorn-utils
    1.33 +
    1.34 +
    1.35 +
    1.36 +
    1.37 +pushd firethorn-utils
    1.38 +
    1.39 +    sudo python3 setup.py install
    1.40 +
    1.41 +popd
    1.42 +
    1.43 +
    1.44 +
    1.45 +pushd firethorn.py
    1.46 +
    1.47 +    sudo python3 setup.py install
    1.48 +
    1.49 +popd
    1.50 +
    1.51 +
    1.52 +slackwebhook=
    1.53 +
    1.54 +
    1.55 +
    1.56 +# The command to run the System checker is:
    1.57 +
    1.58 +python3 -c "import firethorn_utils.system_checker as system_checker;system_checker.main()" -ft=http://tap.roe.ac.uk/firethorn -slack=${slackwebhook}
    1.59 +	
    1.60 +
    1.61 +# Let's add this as a cronjob
    1.62 +
    1.63 +crontab -e
    1.64 +
    1.65 +* * * * * /usr/bin/python3 -c "import firethorn_utils.system_checker as system_checker;system_checker.main()" -ft=http://tap.roe.ac.uk/firethorn -slack=${slackwebhook}
    1.66 +
    1.67 +# This will run the healthchecker every 1 minute
    1.68 +

     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/stv/20200514-TAP-Swarm-deploy-2.1.36.txt	Wed May 20 14:18:48 2020 +0300
     2.3 @@ -0,0 +1,403 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#  
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#
    2.25 +
    2.26 +
    2.27 +## ------------------------------------------------------------------------------------------
    2.28 +## CreateVM
    2.29 +## ------------------------------------------------------------------------------------------
    2.30 +
    2.31 +stv@trop02:~$ createvm
    2.32 +..
    2.33 +INFO : Node name [Araybwyn]
    2.34 +INFO : Base name [fedora-28-32G-docker-base-20181016.qcow]
    2.35 +INFO : Base path [/var/lib/libvirt/images/base/fedora-28-32G-docker-base-20181016.qcow]
    2.36 +INFO : Disc name [Araybwyn.qcow]
    2.37 +INFO : Disc size [32GiB]
    2.38 +
    2.39 +
    2.40 +
    2.41 +
    2.42 +## ------------------------------------------------------------------------------------------
    2.43 +## Initialize Swarm with VM's address
    2.44 +## ------------------------------------------------------------------------------------------
    2.45 +
    2.46 +ssh Stevedore@Araybwyn
    2.47 +
    2.48 +ip=$(ip -f inet -o addr show ens3|cut -d\  -f 7 | cut -d/ -f 1)
    2.49 +docker swarm init --advertise-addr ${ip}
    2.50 +
    2.51 +
    2.52 +## Firewall Ports for Swarm
    2.53 +
    2.54 +sudo su
    2.55 +    firewall-cmd --add-port=2377/tcp --permanent
    2.56 +    firewall-cmd --add-port=2377/tcp --permanent
    2.57 +    firewall-cmd --add-port=7946/tcp --permanent
    2.58 +    firewall-cmd --add-port=4789/tcp --permanent
    2.59 +    firewall-cmd --add-port=4789/udp --permanent
    2.60 +    iptables -A INPUT -p 50 -j ACCEPT    
    2.61 +    firewall-cmd --reload
    2.62 +exit
    2.63 +
    2.64 +
    2.65 +
    2.66 +
    2.67 +## -------------------------------------------------------------------------------------------
    2.68 +## Add secret function.
    2.69 +## -------------------------------------------------------------------------------------------
    2.70 +
    2.71 +    secrethost='stv@shepseskaf.roe.ac.uk'
    2.72 +    secretfile='${HOME:?}/secret.store.20190828'
    2.73 +
    2.74 +    secret()
    2.75 +    {
    2.76 +        local key=${1:?}
    2.77 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    2.78 +    }
    2.79 +
    2.80 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    2.81 +
    2.82 +
    2.83 +
    2.84 +## -------------------------------------------------------------------------------------------
    2.85 +## Set the target branch and builtag
    2.86 +## -------------------------------------------------------------------------------------------
    2.87 +
    2.88 +
    2.89 +    export buildtag=2.1.36
    2.90 +    export branch=default
    2.91 +
    2.92 +
    2.93 +
    2.94 +## -------------------------------------------------------------------------------------------
    2.95 +## Create our chain.properties file
    2.96 +## -------------------------------------------------------------------------------------------
    2.97 +
    2.98 +cat > "${HOME:?}/chain.properties" << EOF
    2.99 +
   2.100 +metadata=postgres
   2.101 +metauser=$(pwgen 20 1)
   2.102 +metapass=$(pwgen 20 1) 
   2.103 +
   2.104 +userhost=$(secret 'firethorn.user.host')
   2.105 +userdata=$(secret 'firethorn.user.data')
   2.106 +useruser=$(secret 'firethorn.user.user')
   2.107 +userpass=$(secret 'firethorn.user.pass')
   2.108 +usertype=mssql
   2.109 +
   2.110 +datahost=$(secret 'firethorn.data.host')
   2.111 +datadata=$(secret 'firethorn.data.data')
   2.112 +datacatalog=$(secret 'firethorn.data.catalog')
   2.113 +datatype=$(secret 'firethorn.data.type')
   2.114 +datauser=$(secret 'firethorn.data.user')
   2.115 +datapass=$(secret 'firethorn.data.pass')
   2.116 +
   2.117 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   2.118 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   2.119 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   2.120 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   2.121 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   2.122 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   2.123 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   2.124 +
   2.125 +tunneluser=$(secret 'ssh.tunnel.user')
   2.126 +tunnelhost=$(secret 'ssh.tunnel.host')
   2.127 +
   2.128 +admingroup=$(pwgen 20 1)	
   2.129 +adminuser=$(pwgen 20 1)
   2.130 +adminpass=$(pwgen 20 1)
   2.131 +
   2.132 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   2.133 + 
   2.134 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   2.135 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   2.136 +
   2.137 +guestgroup=friends
   2.138 +endpoint=$(secret 'endpoint')
   2.139 +
   2.140 +firethornhost=gillian
   2.141 +
   2.142 +EOF
   2.143 +
   2.144 +
   2.145 +
   2.146 +## -----------------------------------------------------
   2.147 +## Create our Firethorn properties.
   2.148 +## -----------------------------------------------------
   2.149 +
   2.150 +source "${HOME:?}/chain.properties"
   2.151 +cat > "${HOME:?}/firethorn.properties" << EOF
   2.152 +
   2.153 +firethorn.meta.type=pgsql
   2.154 +firethorn.meta.driver=org.postgresql.Driver
   2.155 +firethorn.meta.host=bethany
   2.156 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   2.157 +firethorn.meta.data=postgres
   2.158 +firethorn.meta.user=${metauser:?}
   2.159 +firethorn.meta.pass=${metapass:?}
   2.160 +
   2.161 +firethorn.user.type=mssql
   2.162 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   2.163 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   2.164 +firethorn.user.host=${userhost:?}
   2.165 +firethorn.user.data=${userdata:?}
   2.166 +firethorn.user.user=${useruser:?}
   2.167 +firethorn.user.pass=${userpass:?}
   2.168 +
   2.169 +firethorn.tapschema.resource.name=${tapschemajdbc}
   2.170 +firethorn.tapschema.database.name=${tapschemadata}
   2.171 +firethorn.tapschema.database.host=${tapschemahost}
   2.172 +firethorn.tapschema.database.port=${tapschemaport}
   2.173 +firethorn.tapschema.database.user=${tapschemauser}
   2.174 +firethorn.tapschema.database.pass=${tapschemapass}
   2.175 +
   2.176 +firethorn.admin.community=${admingroup:?}
   2.177 +firethorn.admin.user.name=${adminuser:?}
   2.178 +firethorn.admin.user.pass=${adminpass:?}
   2.179 +
   2.180 +firethorn.guest.community=${guestgroup:?}
   2.181 +
   2.182 +adql.query.delay.first=${adql_query_delay_first:?}
   2.183 +
   2.184 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   2.185 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   2.186 +
   2.187 +firethornhost=gillian
   2.188 +
   2.189 +
   2.190 +EOF
   2.191 +
   2.192 +
   2.193 +
   2.194 +## -------------------------------------------------------------------------------------------
   2.195 +## Create our compose env files.
   2.196 +## -------------------------------------------------------------------------------------------
   2.197 +
   2.198 +source "${HOME:?}/chain.properties"
   2.199 +cat > "${HOME:?}/.env" << EOF
   2.200 +
   2.201 +buildtag=${branch:?}
   2.202 +endpoint=${endpoint:?}
   2.203 +
   2.204 +
   2.205 +metadata=postgres
   2.206 +metauser=${metauser:?}
   2.207 +metapass=${metapass:?}
   2.208 +firethorn.meta.type=pgsql
   2.209 +firethorn.meta.driver=org.postgresql.Driver
   2.210 +firethorn.meta.host=bethany
   2.211 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   2.212 +firethorn.meta.data=postgres
   2.213 +firethorn.meta.user=${metauser:?}
   2.214 +firethorn.meta.pass=${metapass:?}
   2.215 +
   2.216 +
   2.217 +datadata=${datadata:?}
   2.218 +datauser=${datauser:?}
   2.219 +datapass=${datapass:?}
   2.220 +datahost=${datahost:?}
   2.221 +datacatalog=${datacatalog:?}
   2.222 +datatype=mssql
   2.223 +
   2.224 +
   2.225 +tapschemadata=${tapschemadata:?}
   2.226 +tapschemauser=${tapschemauser:?}
   2.227 +tapschemapass=${tapschemapass:?}
   2.228 +tapschemahost=${tapschemahost}
   2.229 +tapschemaport=${tapschemaport:?}
   2.230 +tapschemajdbc=${tapschemajdbc:?}
   2.231 +tapschematype=${tapschematype:?}
   2.232 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   2.233 +firethorn.tapschema.database.name=${tapschemadata:?}
   2.234 +firethorn.tapschema.database.host=${tapschemahost:?}
   2.235 +firethorn.tapschema.database.port=5432
   2.236 +firethorn.tapschema.database.user=${tapschemauser:?}
   2.237 +firethorn.tapschema.database.pass=${tapschemapass:?}
   2.238 +firethorn.tapschema.database.type=pgsql
   2.239 +
   2.240 +userhost=${userhost:?}
   2.241 +userdata=${userdata:?}
   2.242 +useruser=${useruser:?}
   2.243 +userpass=${userpass:?}
   2.244 +usertype=mssql
   2.245 +firethorn.user.type=mssql
   2.246 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   2.247 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   2.248 +firethorn.user.host=${userhost:?}
   2.249 +firethorn.user.data=${userdata:?}
   2.250 +firethorn.user.user=${useruser:?}
   2.251 +firethorn.user.pass=${userpass:?}
   2.252 +
   2.253 +
   2.254 +admingroup=${admingroup:?}
   2.255 +adminuser=${adminuser:?}
   2.256 +adminpass=${adminpass:?}
   2.257 +guestgroup=${guestgroup:?}
   2.258 +firethorn.admin.community=${admingroup:?}
   2.259 +firethorn.admin.user.name=${adminuser:?}
   2.260 +firethorn.admin.user.pass=${adminpass:?}
   2.261 +firethorn.guest.community=${guestgroup:?}
   2.262 +
   2.263 +
   2.264 +adql_query_delay_first=${adql_query_delay_first:?}
   2.265 +adql.query.delay.first=${adql_query_delay_first:?}
   2.266 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   2.267 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   2.268 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   2.269 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   2.270 +
   2.271 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   2.272 +firethornhost=gillian
   2.273 +
   2.274 +
   2.275 +EOF
   2.276 +
   2.277 +
   2.278 +source "${HOME:?}/chain.properties"
   2.279 +cat > "${HOME:?}/.tapschema_env" << EOF
   2.280 +
   2.281 +POSTGRES_DB=${tapschemadata}
   2.282 +POSTGRES_USER=${tapschemauser}
   2.283 +POSTGRES_PASSWORD=${tapschemapass}
   2.284 +
   2.285 +EOF
   2.286 +
   2.287 +
   2.288 +source "${HOME:?}/chain.properties"
   2.289 +cat > "${HOME:?}/.metadata_env" << EOF
   2.290 +
   2.291 +POSTGRES_DB=${metadata}
   2.292 +POSTGRES_USER=${metauser}
   2.293 +POSTGRES_PASSWORD=${metapass}
   2.294 +         
   2.295 +EOF
   2.296 +
   2.297 +
   2.298 +
   2.299 +
   2.300 +## -------------------------------------------------------------------------------------------
   2.301 +## Fetch the TAP Deployer
   2.302 +## -------------------------------------------------------------------------------------------
   2.303 +
   2.304 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   2.305 +
   2.306 +
   2.307 +
   2.308 +
   2.309 +## ------------------------------------------------------------------------
   2.310 +## Deploy Swarm stack
   2.311 +## ------------------------------------------------------------------------
   2.312 +
   2.313 +docker stack deploy -c docker-compose.yml ft
   2.314 +
   2.315 +
   2.316 +
   2.317 +## ------------------------------------------------------------------------
   2.318 +## To remove all docker services 
   2.319 +## ------------------------------------------------------------------------
   2.320 +
   2.321 +## docker service rm $(docker service ls -q)
   2.322 +
   2.323 +
   2.324 +
   2.325 +
   2.326 +## ------------------------------------------------------------------------
   2.327 +## To remove all docker services 
   2.328 +## ------------------------------------------------------------------------
   2.329 +
   2.330 +## docker service rm $(docker service ls -q)
   2.331 +
   2.332 +
   2.333 +
   2.334 +## ------------------------------------------------------------------------
   2.335 +## To remove stack of services
   2.336 +## ------------------------------------------------------------------------
   2.337 +
   2.338 +## docker stack rm ft
   2.339 +
   2.340 +
   2.341 +## ------------------------------------------------------------------------
   2.342 +## Check that the services started
   2.343 +## ------------------------------------------------------------------------
   2.344 +
   2.345 +docker ps
   2.346 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                  PORTS               NAMES
   2.347 +b6c43b0fcc39        firethorn/firethorn-py:latest   "python3"                11 hours ago        Up 11 hours                                 ft_firethorn-py.1.ttyojmr0tw108b9kurg5m1l6t
   2.348 +d347cc0e8fcf        firethorn/firethorn:latest      "/bin/sh -c '/var/lo…"   11 hours ago        Up 11 hours (healthy)   8080/tcp            ft_gillian.1.7gjs4uwpxzx8d55agqd91e60v
   2.349 +e81cc914579c        firethorn/ogsadai:latest        "/bin/sh -c '/var/lo…"   11 hours ago        Up 11 hours (healthy)   8080/tcp            ft_jarmila.1.sy5d8zm2f4f6oehp46h0f4lxy
   2.350 +eadf4ace83d4        firethorn/postgres:latest       "docker-entrypoint.s…"   11 hours ago        Up 11 hours             5432/tcp            ft_carolina.1.mbikmqmtgg9r5joehq0ywz0xp
   2.351 +7c4fa13ff87e        firethorn/postgres:latest       "docker-entrypoint.s…"   11 hours ago        Up 11 hours             5432/tcp            ft_bethany.1.sope4plrrq51uctwvdod4w078
   2.352 +
   2.353 +
   2.354 +## ------------------------------------------------------------------------
   2.355 +## Run Docker Compose tap-deployer.yml from firethorn.py
   2.356 +## ------------------------------------------------------------------------
   2.357 +
   2.358 +docker exec -it ft_firethorn-py.1.ttyojmr0tw108b9kurg5m1l6t bash
   2.359 +
   2.360 +
   2.361 +cat > "deployer.py" << EOF
   2.362 +
   2.363 +def main():
   2.364 +    import firethorn
   2.365 +    import firethorn_utils.configurator as configur
   2.366 +    ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   2.367 +    ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   2.368 +    configurator =  configur.Configurator(ft)
   2.369 +    configurator.load_resources("https://raw.githubusercontent.com/stvoutsin/metadata/master/firethorn/config/osa-tap.json")
   2.370 +    configurator.load_resources("https://raw.githubusercontent.com/stvoutsin/metadata/master/firethorn/config/ssa-tap.json")
   2.371 +    configurator.load_resources("https://raw.githubusercontent.com/stvoutsin/metadata/master/firethorn/config/vsa-tap.json")
   2.372 +    configurator.load_resources("https://raw.githubusercontent.com/stvoutsin/metadata/master/firethorn/config/wsa-tap.json")
   2.373 +
   2.374 +if __name__== "__main__":
   2.375 +    main()
   2.376 +
   2.377 +EOF
   2.378 +
   2.379 +
   2.380 +
   2.381 +nohup python3 -u deployer.py &
   2.382 +
   2.383 +## ------------------------------------------------------------------------
   2.384 +## Update Apache Proxy with new TAP services
   2.385 +## ------------------------------------------------------------------------
   2.386 +
   2.387 +
   2.388 +    ProxyPassMatch    ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.389 +    ProxyPassReverse  ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.390 +
   2.391 +    ProxyPassMatch    ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   2.392 +    ProxyPassReverse  ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   2.393 +
   2.394 +    ProxyPassMatch    ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   2.395 +    ProxyPassReverse  ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   2.396 +
   2.397 +    ProxyPassMatch    ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   2.398 +    ProxyPassReverse  ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   2.399 +
   2.400 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   2.401 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   2.402 +
   2.403 +
   2.404 +
   2.405 +
   2.406 +

     3.1 Binary file doc/presentations/stelios/20180415-Asterics-Edinburgh/firethorn-py.odg has changed