1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190626-Firethorn-Swarm.txt	Wed Jun 26 17:17:44 2019 +0100
     1.3 @@ -0,0 +1,402 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2015, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#  
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +
    1.27 +## ------------------------------------------------------------------------------------------
    1.28 +## Create Master VM 
    1.29 +## ------------------------------------------------------------------------------------------
    1.30 +
    1.31 +
    1.32 +createvm Cadelicia (master)
    1.33 +..
    1.34 +Domain Cadelicia started
    1.35 +
    1.36 +ssh Stevedore@Cadelicia
    1.37 +
    1.38 +
    1.39 +
    1.40 +## ------------------------------------------------------------------------------------------
    1.41 +## Initialize Swarm with VM's address
    1.42 +## ------------------------------------------------------------------------------------------
    1.43 +
    1.44 +[Stevedore@Cadelicia ~]$ ifconfig
    1.45 +
    1.46 +       ..  192.168.201.11  ..
    1.47 +
    1.48 +
    1.49 +docker swarm init --advertise-addr 192.168.201.11  
    1.50 +
    1.51 +
    1.52 +		Swarm initialized: current node (5zrs2po0x1teeyzigf09mw5vi) is now a manager.
    1.53 +
    1.54 +		To add a worker to this swarm, run the following command:
    1.55 +
    1.56 +		    docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
    1.57 +
    1.58 +		To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
    1.59 +
    1.60 +
    1.61 +## ------------------------------------------------------------------------------------------
    1.62 +## Create Spark-net network
    1.63 +## ------------------------------------------------------------------------------------------
    1.64 +## docker network create --driver overlay firethorn-net
    1.65 +
    1.66 +
    1.67 +    ## Fix Firewall issue
    1.68 +    sudo su
    1.69 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.70 +	    firewall-cmd --add-port=2377/tcp --permanent
    1.71 +	    firewall-cmd --add-port=7946/tcp --permanent
    1.72 +	    firewall-cmd --add-port=4789/tcp --permanent
    1.73 +	    firewall-cmd --add-port=4789/udp --permanent
    1.74 +	    iptables -A INPUT -p 50 -j ACCEPT    
    1.75 +	    firewall-cmd --reload
    1.76 +    exit
    1.77 +
    1.78 +
    1.79 +## -------------------------------------------------------------------------------------------
    1.80 +## Add secret function.
    1.81 +## -------------------------------------------------------------------------------------------
    1.82 +
    1.83 +    secrethost='stv@shepseskaf.roe.ac.uk'
    1.84 +    secretfile='${HOME:?}/secret.store.20181212'
    1.85 +
    1.86 +    secret()
    1.87 +    {
    1.88 +        local key=${1:?}
    1.89 +        ssh -o 'VisualHostKey=no' "${secrethost:?}" "sed -n 's/${key}=\\(.*\\)/\\1/p' \"${secretfile:?}\"" 
    1.90 +    }
    1.91 +
    1.92 +    secret 'firethorn.ssh.keys' >> "${HOME}/.ssh/authorized_keys"
    1.93 +
    1.94 +
    1.95 +
    1.96 +## -------------------------------------------------------------------------------------------
    1.97 +## Set the target branch and builtag
    1.98 +## -------------------------------------------------------------------------------------------
    1.99 +
   1.100 +
   1.101 +    export buildtag=2.1.33
   1.102 +    export branch=default
   1.103 +
   1.104 +
   1.105 +
   1.106 +## -------------------------------------------------------------------------------------------
   1.107 +## Create our chain.properties file
   1.108 +## -------------------------------------------------------------------------------------------
   1.109 +
   1.110 +cat > "${HOME:?}/chain.properties" << EOF
   1.111 +
   1.112 +metadata=postgres
   1.113 +metauser=$(pwgen 20 1)
   1.114 +metapass=$(pwgen 20 1) 
   1.115 +
   1.116 +userhost=$(secret 'firethorn.user.host')
   1.117 +userdata=$(secret 'firethorn.user.data')
   1.118 +useruser=$(secret 'firethorn.user.user')
   1.119 +userpass=$(secret 'firethorn.user.pass')
   1.120 +usertype=mssql
   1.121 +
   1.122 +datahost=$(secret 'firethorn.data.host')
   1.123 +datadata=$(secret 'firethorn.data.data')
   1.124 +datacatalog=$(secret 'firethorn.data.catalog')
   1.125 +datatype=$(secret 'firethorn.data.type')
   1.126 +datauser=$(secret 'firethorn.data.user')
   1.127 +datapass=$(secret 'firethorn.data.pass')
   1.128 +
   1.129 +tapschemadata=$(secret 'firethorn.tapschema.database.name')
   1.130 +tapschemauser=$(secret 'firethorn.tapschema.database.user')
   1.131 +tapschemapass=$(secret 'firethorn.tapschema.database.pass')
   1.132 +tapschemahost=$(secret 'firethorn.tapschema.database.host')
   1.133 +tapschemaport=$(secret 'firethorn.tapschema.database.port')
   1.134 +tapschematype=$(secret 'firethorn.tapschema.database.type')
   1.135 +tapschemajdbc=$(secret 'firethorn.tapschema.jdbc.name')
   1.136 +
   1.137 +tunneluser=$(secret 'ssh.tunnel.user')
   1.138 +tunnelhost=$(secret 'ssh.tunnel.host')
   1.139 +
   1.140 +admingroup=$(pwgen 20 1)	
   1.141 +adminuser=$(pwgen 20 1)
   1.142 +adminpass=$(pwgen 20 1)
   1.143 +
   1.144 +adql_query_delay_first=$(secret 'adql.query.delay.first')
   1.145 + 
   1.146 +firethorn_limits_rows_absolute=$(secret 'firethorn.limits.rows.absolute')
   1.147 +firethorn_limits_rows_default=$(secret 'firethorn.limits.rows.default')
   1.148 +
   1.149 +guestgroup=friends
   1.150 +endpoint=$(secret 'endpoint')
   1.151 +
   1.152 +EOF
   1.153 +
   1.154 +
   1.155 +
   1.156 +## -----------------------------------------------------
   1.157 +## Create our Firethorn properties.
   1.158 +## -----------------------------------------------------
   1.159 +
   1.160 +source "${HOME:?}/chain.properties"
   1.161 +cat > "${HOME:?}/firethorn.properties" << EOF
   1.162 +
   1.163 +firethorn.meta.type=pgsql
   1.164 +firethorn.meta.driver=org.postgresql.Driver
   1.165 +firethorn.meta.host=bethany
   1.166 +firethorn.meta.url=jdbc:postgresql://bethany/${metadata}
   1.167 +firethorn.meta.data=postgres
   1.168 +firethorn.meta.user=${metauser:?}
   1.169 +firethorn.meta.pass=${metapass:?}
   1.170 +
   1.171 +firethorn.user.type=mssql
   1.172 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.173 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.174 +firethorn.user.host=${userhost:?}
   1.175 +firethorn.user.data=${userdata:?}
   1.176 +firethorn.user.user=${useruser:?}
   1.177 +firethorn.user.pass=${userpass:?}
   1.178 +
   1.179 +firethorn.tapschema.resource.name=${tapschemajdbc}
   1.180 +firethorn.tapschema.database.name=${tapschemadata}
   1.181 +firethorn.tapschema.database.host=${tapschemahost}
   1.182 +firethorn.tapschema.database.port=${tapschemaport}
   1.183 +firethorn.tapschema.database.user=${tapschemauser}
   1.184 +firethorn.tapschema.database.pass=${tapschemapass}
   1.185 +
   1.186 +firethorn.admin.community=${admingroup:?}
   1.187 +firethorn.admin.user.name=${adminuser:?}
   1.188 +firethorn.admin.user.pass=${adminpass:?}
   1.189 +
   1.190 +firethorn.guest.community=${guestgroup:?}
   1.191 +
   1.192 +adql.query.delay.first=${adql_query_delay_first:?}
   1.193 +
   1.194 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.195 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.196 +	
   1.197 +
   1.198 +EOF
   1.199 +
   1.200 +
   1.201 +
   1.202 +## -------------------------------------------------------------------------------------------
   1.203 +## Create our compose env files.
   1.204 +## -------------------------------------------------------------------------------------------
   1.205 +
   1.206 +source "${HOME:?}/chain.properties"
   1.207 +cat > "${HOME:?}/.env" << EOF
   1.208 +
   1.209 +buildtag=${branch:?}
   1.210 +endpoint=${endpoint:?}
   1.211 +
   1.212 +
   1.213 +metadata=postgres
   1.214 +metauser=${metauser:?}
   1.215 +metapass=${metapass:?}
   1.216 +firethorn.meta.type=pgsql
   1.217 +firethorn.meta.driver=org.postgresql.Driver
   1.218 +firethorn.meta.host=bethany
   1.219 +firethorn.meta.url=jdbc:postgresql://bethany/postgres
   1.220 +firethorn.meta.data=postgres
   1.221 +firethorn.meta.user=${metauser:?}
   1.222 +firethorn.meta.pass=${metapass:?}
   1.223 +
   1.224 +
   1.225 +datadata=${datadata:?}
   1.226 +datauser=${datauser:?}
   1.227 +datapass=${datapass:?}
   1.228 +datahost=${datahost:?}
   1.229 +datacatalog=${datacatalog:?}
   1.230 +datatype=mssql
   1.231 +
   1.232 +
   1.233 +tapschemadata=${tapschemadata:?}
   1.234 +tapschemauser=${tapschemauser:?}
   1.235 +tapschemapass=${tapschemapass:?}
   1.236 +tapschemahost=${tapschemahost}
   1.237 +tapschemaport=${tapschemaport:?}
   1.238 +tapschemajdbc=${tapschemajdbc:?}
   1.239 +tapschematype=${tapschematype:?}
   1.240 +firethorn.tapschema.resource.name=${tapschemajdbc:?}
   1.241 +firethorn.tapschema.database.name=${tapschemadata:?}
   1.242 +firethorn.tapschema.database.host=${tapschemahost:?}
   1.243 +firethorn.tapschema.database.port=5432
   1.244 +firethorn.tapschema.database.user=${tapschemauser:?}
   1.245 +firethorn.tapschema.database.pass=${tapschemapass:?}
   1.246 +firethorn.tapschema.database.type=pgsql
   1.247 +
   1.248 +userhost=${userhost:?}
   1.249 +userdata=${userdata:?}
   1.250 +useruser=${useruser:?}
   1.251 +userpass=${userpass:?}
   1.252 +usertype=mssql
   1.253 +firethorn.user.type=mssql
   1.254 +firethorn.user.url=jdbc:jtds:sqlserver://${userhost:?}/${userdata:?}
   1.255 +firethorn.user.driver=net.sourceforge.jtds.jdbc.Driver
   1.256 +firethorn.user.host=${userhost:?}
   1.257 +firethorn.user.data=${userdata:?}
   1.258 +firethorn.user.user=${useruser:?}
   1.259 +firethorn.user.pass=${userpass:?}
   1.260 +
   1.261 +
   1.262 +admingroup=${admingroup:?}
   1.263 +adminuser=${adminuser:?}
   1.264 +adminpass=${adminpass:?}
   1.265 +guestgroup=${guestgroup:?}
   1.266 +firethorn.admin.community=${admingroup:?}
   1.267 +firethorn.admin.user.name=${adminuser:?}
   1.268 +firethorn.admin.user.pass=${adminpass:?}
   1.269 +firethorn.guest.community=${guestgroup:?}
   1.270 +
   1.271 +
   1.272 +adql_query_delay_first=${adql_query_delay_first:?}
   1.273 +adql.query.delay.first=${adql_query_delay_first:?}
   1.274 +firethorn_limits_rows_absolute=${firethorn_limits_rows_absolute:?}
   1.275 +firethorn_limits_rows_default=${firethorn_limits_rows_default:?}
   1.276 +firethorn.limits.rows.absolute=${firethorn_limits_rows_absolute:?}
   1.277 +firethorn.limits.rows.default=${firethorn_limits_rows_default:?}
   1.278 +
   1.279 +firethorn.ogsadai.endpoint=http://jarmila:8080/ogsadai/services
   1.280 +
   1.281 +
   1.282 +EOF
   1.283 +
   1.284 +
   1.285 +source "${HOME:?}/chain.properties"
   1.286 +cat > "${HOME:?}/.tapschema_env" << EOF
   1.287 +
   1.288 +POSTGRES_DB=${tapschemadatabase}
   1.289 +POSTGRES_USER=${tapschemauser}
   1.290 +POSTGRES_PASSWORD=${tapschemapass}
   1.291 +
   1.292 +EOF
   1.293 +
   1.294 +
   1.295 +source "${HOME:?}/chain.properties"
   1.296 +cat > "${HOME:?}/.metadata_env" << EOF
   1.297 +
   1.298 +POSTGRES_DB=${metadata}
   1.299 +POSTGRES_USER=${metauser}
   1.300 +POSTGRES_PASSWORD=${metapass}
   1.301 +         
   1.302 +EOF
   1.303 +
   1.304 +
   1.305 +## -------------------------------------------------------------------------------------------
   1.306 +## Fetch the TAP Deployer
   1.307 +## -------------------------------------------------------------------------------------------
   1.308 +
   1.309 +wget https://raw.githubusercontent.com/stvoutsin/firethorn.py/master/docker/firethorn-compose-v3.yml -O  docker-compose.yml
   1.310 +
   1.311 +
   1.312 +
   1.313 +## ------------------------------------------------------------------------
   1.314 +## Deploy Swarm stack
   1.315 +## ------------------------------------------------------------------------
   1.316 +
   1.317 +docker stack deploy -c docker-compose.yml ft
   1.318 +
   1.319 +
   1.320 +
   1.321 +## ------------------------------------------------------------------------
   1.322 +## To remove all docker services 
   1.323 +## ------------------------------------------------------------------------
   1.324 +
   1.325 +## docker service rm $(docker service ls -q)
   1.326 +
   1.327 +
   1.328 +
   1.329 +## ------------------------------------------------------------------------
   1.330 +## To remove stack of services
   1.331 +## ------------------------------------------------------------------------
   1.332 +
   1.333 +## docker stack rm ft
   1.334 +
   1.335 +
   1.336 +
   1.337 +## ------------------------------------------------------------------------
   1.338 +## Run Docker Compose tap-deployer.yml from firethorn.py
   1.339 +## ------------------------------------------------------------------------
   1.340 +
   1.341 +
   1.342 +source "${HOME:?}/chain.properties"
   1.343 +
   1.344 +docker-compose --file "docker-compose.yml" run firethorn-py
   1.345 +
   1.346 +import firethorn
   1.347 +import firethorn_utils.configurator as configur
   1.348 +ft = firethorn.Firethorn(endpoint="http://gillian:8080/firethorn")
   1.349 +ft.login(firethorn.config.adminuser, firethorn.config.adminpass, firethorn.config.admingroup)
   1.350 +configurator =  configur.Configurator(ft)
   1.351 +configurator.load_resources("/home/firethorn.py/firethorn/data/osa-tap.json")
   1.352 +
   1.353 +..
   1.354 +Importing JDBC Schema: ATLASDR1
   1.355 +Importing JDBC Schema: ATLASDR2
   1.356 +Importing JDBC Schema: ATLASDR3
   1.357 +Importing JDBC Schema: 2MASS
   1.358 +Importing JDBC Schema: 2XMM
   1.359 +Importing JDBC Schema: WISE
   1.360 +Importing JDBC Schema: VIKINGDR3
   1.361 +Importing JDBC Schema: VHSDR1
   1.362 +Importing JDBC Schema: MGC
   1.363 +Importing JDBC Schema: GLIMPSE
   1.364 +Importing JDBC Schema: FIRST
   1.365 +Importing JDBC Schema: DENIS
   1.366 +TAP Service available at: http://gillian:8080/firethorn/tap/54/
   1.367 +
   1.368 +..
   1.369 +
   1.370 +## ------------------------------------------------------------------------
   1.371 +## [OPTIONAL] Adding a seconf VM to the Swarm
   1.372 +## ------------------------------------------------------------------------
   1.373 +
   1.374 +
   1.375 +## Create Worker VM 
   1.376 +createvm Erennon (worker)
   1.377 +Domain Erennon started
   1.378 +
   1.379 +ssh Stevedore@Erennon
   1.380 +
   1.381 +     docker swarm join --token SWMTKN-1-04u77kiexc4jjh40ncaov4kxvez0pszz8gjoake56i9liplcfb-cnbjy5d7ul9qdd9r307rt9zbv 192.168.201.11:2377
   1.382 +      ...
   1.383 +      This node joined a swarm as a worker.
   1.384 +      ...
   1.385 +
   1.386 +    
   1.387 +     ## Fix Firewall issue
   1.388 +     sudo su
   1.389 +     firewall-cmd --add-port=2377/tcp --permanent
   1.390 +     firewall-cmd --add-port=2377/tcp --permanent
   1.391 +     firewall-cmd --add-port=7946/tcp --permanent
   1.392 +     firewall-cmd --add-port=4789/udp --permanent
   1.393 +     firewall-cmd --add-port=4789/tcp --permanent
   1.394 +     iptables -A INPUT -p 50 -j ACCEPT    
   1.395 +     firewall-cmd --reload
   1.396 +     exit
   1.397 +
   1.398 +
   1.399 +## ------------------------------------------------------------------------
   1.400 +##  Scaling: If we were to scale the cluster, we Could do so as:
   1.401 +## ------------------------------------------------------------------------
   1.402 +docker service scale ft_gillian=4
   1.403 +docker service scale ft_jarmila=4
   1.404 +
   1.405 +## But this will not do anything at the moment