firethorn

changeset 4310:7345ff52db44

Notes on network debug and fix
author Dave Morris <dmr@roe.ac.uk>
date Tue Sep 01 01:31:31 2020 +0100 (15 months ago)
parents cee138223527
children ba9eff2eed8a 44e00a536440
files doc/notes/zrq/20200828-01-vm-shutdown.txt doc/notes/zrq/20200831-01-recovery.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/zrq/20200828-01-vm-shutdown.txt	Tue Sep 01 01:31:31 2020 +0100
     1.3 @@ -0,0 +1,217 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#zrq-notes-time
    1.25 +#zrq-notes-indent
    1.26 +#zrq-notes-crypto
    1.27 +#zrq-notes-ansible
    1.28 +#zrq-notes-osformat
    1.29 +#
    1.30 +
    1.31 +
    1.32 +trop01
    1.33 +
    1.34 +    # Firethorn services
    1.35 +
    1.36 +    source ${HOME}/libvirt.settings
    1.37 +    virsh -c ${libvirtcon} list
    1.38 +
    1.39 +--START--
    1.40 + Id    Name                           State
    1.41 +----------------------------------------------------
    1.42 + 41    Froeseth                       running
    1.43 + 43    Delild                         running
    1.44 + 49    Cadelicia                      running
    1.45 + 50    Astoalith                      running
    1.46 +--END--
    1.47 +
    1.48 +    source ${HOME}/libvirt.settings
    1.49 +    virsh -c ${libvirtcon} list --all
    1.50 +
    1.51 +--START--
    1.52 + Id    Name                           State
    1.53 +----------------------------------------------------
    1.54 + -     Astoalith                      shut off
    1.55 + -     Cadelicia                      shut off
    1.56 + -     Delild                         shut off
    1.57 + -     Froeseth                       shut off
    1.58 +--END--
    1.59 +
    1.60 +
    1.61 +trop02
    1.62 +
    1.63 +    source ${HOME}/libvirt.settings
    1.64 +    virsh -c ${libvirtcon} list
    1.65 +
    1.66 +    # Firethorn services
    1.67 +    # Plus some test VMs ?
    1.68 +
    1.69 +--START--
    1.70 + Id    Name                           State
    1.71 +----------------------------------------------------
    1.72 + 3     Acilamwen                      running
    1.73 + 36    Ulov                           running
    1.74 + 45    Lothigometh                    running
    1.75 + 46    Araybwyn                       running
    1.76 +--END--
    1.77 +
    1.78 +    source ${HOME}/libvirt.settings
    1.79 +    virsh -c ${libvirtcon} list --all
    1.80 +
    1.81 +--START--
    1.82 + Id    Name                           State
    1.83 +----------------------------------------------------
    1.84 + -     Acilamwen                      shut off
    1.85 + -     Araybwyn                       shut off
    1.86 + -     Ibalehar                       shut off
    1.87 + -     Lothigometh                    shut off
    1.88 + -     Ulov                           shut off
    1.89 +--END--
    1.90 +
    1.91 +
    1.92 +trop03
    1.93 +
    1.94 +    source ${HOME}/libvirt.settings
    1.95 +    virsh -c ${libvirtcon} list
    1.96 +
    1.97 +    # Kafka service, with one node on trop04
    1.98 +    # nodenames has a list of who is who
    1.99 +
   1.100 +--START--
   1.101 + Id    Name                           State
   1.102 +----------------------------------------------------
   1.103 + 86    Fosauri                        running
   1.104 + 87    Marpus                         running
   1.105 + 90    Edwalafia                      running
   1.106 + 91    Angece                         running
   1.107 + 92    Onoza                          running
   1.108 + 93    Stedigo                        running
   1.109 + 106   Umiawyth                       running
   1.110 +--END--
   1.111 +
   1.112 +    source ${HOME}/libvirt.settings
   1.113 +    virsh -c ${libvirtcon} list --all
   1.114 +
   1.115 +--START--
   1.116 + Id    Name                           State
   1.117 +----------------------------------------------------
   1.118 + -     Angece                         shut off
   1.119 + -     Edwalafia                      shut off
   1.120 + -     Fosauri                        shut off
   1.121 + -     Marpus                         shut off
   1.122 + -     Onoza                          shut off
   1.123 + -     Stedigo                        shut off
   1.124 + -     Umiawyth                       shut off
   1.125 +--END--
   1.126 +
   1.127 +trop04
   1.128 +
   1.129 +    # Sendalin - Zookeeper service
   1.130 +    # Wumar - test service
   1.131 +
   1.132 +    source ${HOME}/libvirt.settings
   1.133 +    virsh -c ${libvirtcon} list
   1.134 +
   1.135 +--START--
   1.136 + Id    Name                           State
   1.137 +----------------------------------------------------
   1.138 + 18    Sendalin                       running
   1.139 + 25    Wumar                          running
   1.140 +--END--
   1.141 +
   1.142 +
   1.143 +    source ${HOME}/libvirt.settings
   1.144 +    virsh -c ${libvirtcon} list --all
   1.145 +
   1.146 +--START--
   1.147 + Id    Name                           State
   1.148 +----------------------------------------------------
   1.149 + -     Sendalin                       shut off
   1.150 + -     Wumar                          shut off
   1.151 +--END--
   1.152 +
   1.153 +
   1.154 +# -----------------------------------------------------
   1.155 +# [firethorn@shep]
   1.156 +
   1.157 +    source ${HOME}/libvirt.env
   1.158 +    virsh -c ${libvirtcon:?} list
   1.159 +
   1.160 +--START--
   1.161 + Id    Name                           State
   1.162 +----------------------------------------------------
   1.163 + 1     pyratest3                      running
   1.164 + 2     pyratest1                      running
   1.165 + 3     pyratest2                      running
   1.166 +--END--
   1.167 +
   1.168 +
   1.169 +    virsh -c ${libvirtcon:?} shutdown pyratest1
   1.170 +    virsh -c ${libvirtcon:?} shutdown pyratest2
   1.171 +    virsh -c ${libvirtcon:?} shutdown pyratest3
   1.172 +
   1.173 +
   1.174 +    source ${HOME}/libvirt.env
   1.175 +    virsh -c ${libvirtcon} list --all
   1.176 +
   1.177 +--START--
   1.178 + Id    Name                           State
   1.179 +----------------------------------------------------
   1.180 + -     amarna                         shut off
   1.181 + -     pyratest1                      shut off
   1.182 + -     pyratest2                      shut off
   1.183 + -     pyratest3                      shut off
   1.184 +--END--
   1.185 +
   1.186 +
   1.187 +
   1.188 +
   1.189 +# -----------------------------------------------------
   1.190 +# [firethorn@djer]
   1.191 +
   1.192 +    source ${HOME}/libvirt.env
   1.193 +    virsh -c ${libvirtcon} list
   1.194 +
   1.195 +--START--
   1.196 + Id   Name         State
   1.197 +----------------------------
   1.198 + 1    Coccinea     running
   1.199 + 3    Pyracantha   running
   1.200 +--END--
   1.201 +
   1.202 +    virsh -c ${libvirtcon:?} shutdown Coccinea
   1.203 +    virsh -c ${libvirtcon:?} shutdown Pyracantha
   1.204 +
   1.205 +
   1.206 +    virsh -c $libvirtcon list --all
   1.207 +
   1.208 +--START--
   1.209 + Id   Name                State
   1.210 +------------------------------------
   1.211 + -    Coccinea            shut off
   1.212 + -    Coccinea-backup     shut off
   1.213 + -    malus               shut off
   1.214 + -    Pyracantha          shut off
   1.215 + -    Pyracantha-backup   shut off
   1.216 + -    pyrus               shut off
   1.217 +--END--
   1.218 +
   1.219 +
   1.220 +
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/doc/notes/zrq/20200831-01-recovery.txt	Tue Sep 01 01:31:31 2020 +0100
     2.3 @@ -0,0 +1,579 @@
     2.4 +#
     2.5 +# <meta:header>
     2.6 +#   <meta:licence>
     2.7 +#     Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
     2.8 +#
     2.9 +#     This information is free software: you can redistribute it and/or modify
    2.10 +#     it under the terms of the GNU General Public License as published by
    2.11 +#     the Free Software Foundation, either version 3 of the License, or
    2.12 +#     (at your option) any later version.
    2.13 +#
    2.14 +#     This information is distributed in the hope that it will be useful,
    2.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    2.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    2.17 +#     GNU General Public License for more details.
    2.18 +#
    2.19 +#     You should have received a copy of the GNU General Public License
    2.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    2.21 +#   </meta:licence>
    2.22 +# </meta:header>
    2.23 +#
    2.24 +#zrq-notes-time
    2.25 +#zrq-notes-indent
    2.26 +#zrq-notes-crypto
    2.27 +#zrq-notes-ansible
    2.28 +#zrq-notes-osformat
    2.29 +#
    2.30 +
    2.31 +# -----------------------------------------------------
    2.32 +# Check what is running on the Firethorn VM.
    2.33 +#[Stevedore@Araybwyn]
    2.34 +
    2.35 +    docker ps -a
    2.36 +
    2.37 +--START--
    2.38 +CONTAINER ID        IMAGE                           COMMAND                  CREATED             STATUS                     PORTS               NAMES
    2.39 +a35be4a47a03        firethorn/firethorn:2.1.36      "/bin/sh -c '/var/lo…"   2 days ago          Up 2 days (healthy)        8080/tcp            ft_gillian.1.x07m4qlin6a7onczfp3hbcevx
    2.40 +bff3f7d8343e        firethorn/postgres:2.1.36       "docker-entrypoint.s…"   2 days ago          Up 2 days                  5432/tcp            ft_carolina.1.kq9krov9tvlt1tvtyhvgi6xbd
    2.41 +2d1c14fa2631        firethorn/postgres:2.1.36       "docker-entrypoint.s…"   2 days ago          Up 2 days                  5432/tcp            ft_bethany.1.4o8yg7e9zhuf404vhz2m8yt68
    2.42 +4732f012f6c3        firethorn/ogsadai:2.1.36        "/bin/sh -c '/var/lo…"   2 days ago          Up 2 days (healthy)        8080/tcp            ft_jarmila.1.lpfsezv3cyw4cgz6zrytqlbdp
    2.43 +fa88eb69066f        firethorn/firethorn-py:latest   "python3"                2 days ago          Up 2 days                                      ft_firethorn-py.1.sdusyp77tns4g2nouo0wi3v9k
    2.44 +969ec42ca52e        firethorn/firethorn:2.1.36      "/bin/sh -c '/var/lo…"   9 days ago          Exited (143) 3 days ago                        ft_gillian.1.f39h27599vgj076xmalat1hfo
    2.45 +9dae93eccbf4        firethorn/firethorn:2.1.36      "/bin/sh -c '/var/lo…"   11 days ago         Exited (137) 10 days ago                       ft_gillian.1.q1l043x6unlle57vlp2e0o9cy
    2.46 +1ec21ceeb354        firethorn/firethorn:2.1.36      "/bin/sh -c '/var/lo…"   13 days ago         Exited (137) 11 days ago                       ft_gillian.1.ca9jet9o3iyxlikl14pkdabbo
    2.47 +9fc8c04ae5d3        firethorn/firethorn:2.1.36      "/bin/sh -c '/var/lo…"   3 weeks ago         Exited (143) 13 days ago                       ft_gillian.1.uj09twzov0em2hqt8g152n2rb
    2.48 +80d5561cf7a3        firethorn/firethorn-py:latest   "python3"                4 weeks ago         Dead                                           ft_firethorn-py.1.qyiv0gx4ygmb7givp94s94hqz
    2.49 +15bb7d5287bd        284797198230                    "bash"                   4 weeks ago         Exited (0) 4 weeks ago                         stevedore_builder_run_2
    2.50 +d3422fdb6110        9c51267fe107                    "bash"                   4 weeks ago         Exited (130) 4 weeks ago                       stevedore_builder_run_1
    2.51 +--END--
    2.52 +
    2.53 +# -----------------------------------------------------
    2.54 +# Checking services on the Firethorn VM.
    2.55 +#[Stevedore@Araybwyn]
    2.56 +
    2.57 +    curl 'http://192.168.202.8:8080/firethorn/adql/resource/select'
    2.58 +
    2.59 +--START--
    2.60 +[
    2.61 +    {
    2.62 +    "type":"http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json",
    2.63 +    "vosi":"http://192.168.202.8:8080/firethorn/adql/resource/54/vosi",....
    2.64 +....
    2.65 +....
    2.66 +....
    2.67 +    }
    2.68 +]
    2.69 +--END--
    2.70 +
    2.71 +
    2.72 +    curl --head 'http://192.168.202.8:8080/firethorn/adql/resource/select'
    2.73 +
    2.74 +--START--
    2.75 +HTTP/1.1 200
    2.76 +X-Clacks-Overhead: GNU Terry Pratchett
    2.77 +firethorn.auth.username: anon-R22BNZT2EQFJWAAAAF2EKKYLDE
    2.78 +firethorn.auth.community: friends
    2.79 +Content-Type: application/json;charset=UTF-8
    2.80 +Transfer-Encoding: chunked
    2.81 +Date: Mon, 31 Aug 2020 15:38:02 GMT
    2.82 +--END--
    2.83 +
    2.84 +
    2.85 +    curl --head 'http://localhost:8080/firethorn/adql/resource/select'
    2.86 +
    2.87 +--START--
    2.88 +HANGS
    2.89 +--END--
    2.90 +
    2.91 +
    2.92 +    curl --head 'http://araybwyn:8080/firethorn/adql/resource/select'
    2.93 +
    2.94 +--START--
    2.95 +HTTP/1.1 200
    2.96 +X-Clacks-Overhead: GNU Terry Pratchett
    2.97 +firethorn.auth.username: anon-YTSRYHYSJ5T6KAAAAF2EKLXUFM
    2.98 +firethorn.auth.community: friends
    2.99 +Content-Type: application/json;charset=UTF-8
   2.100 +Transfer-Encoding: chunked
   2.101 +Date: Mon, 31 Aug 2020 15:42:19 GMT
   2.102 +--END--
   2.103 +
   2.104 +    #
   2.105 +    # Suggests the service is listening on / mapped to external interface
   2.106 +    # on the host VM, but not to the localhost interface on the host VM.
   2.107 +    #
   2.108 +
   2.109 +
   2.110 +# -----------------------------------------------------
   2.111 +# Checking services on the Apache VM.
   2.112 +#[Stevedore@Araybwyn]
   2.113 +
   2.114 +    curl --head 'http://araybwyn:8080/firethorn/adql/resource/select'
   2.115 +
   2.116 +--START--
   2.117 +HTTP/1.1 200
   2.118 +X-Clacks-Overhead: GNU Terry Pratchett
   2.119 +firethorn.auth.username: anon-U46NFPNOPRKFUAAAAF2EKMMLTA
   2.120 +firethorn.auth.community: friends
   2.121 +Content-Type: application/json;charset=UTF-8
   2.122 +Transfer-Encoding: chunked
   2.123 +Date: Mon, 31 Aug 2020 15:45:09 GMT
   2.124 +--END--
   2.125 +
   2.126 +
   2.127 +# -----------------------------------------------------
   2.128 +# Check what is running on the Apache VM.
   2.129 +#[Stevedore@Araybwyn]
   2.130 +
   2.131 +    docker ps -a
   2.132 +
   2.133 +--START--
   2.134 +CONTAINER ID        IMAGE                     COMMAND                  CREATED             STATUS                      PORTS                NAMES
   2.135 +fc403b749771        firethorn/apache:2.1.25   "/usr/local/bin/http…"   25 minutes ago      Exited (0) 16 minutes ago                        apache2
   2.136 +bc8d6979e451        firethorn/apache:2.1.25   "/usr/local/bin/http…"   24 months ago       Up 16 minutes               0.0.0.0:80->80/tcp   apache
   2.137 +--END--
   2.138 +
   2.139 +
   2.140 +# -----------------------------------------------------
   2.141 +# Check the Apache logs.
   2.142 +#[Stevedore@Araybwyn]
   2.143 +
   2.144 +    docker exec -it apache bash
   2.145 +
   2.146 +        tail -f -n 1000 /var/log/httpd/tap.roe.ac.uk-error.log
   2.147 +
   2.148 +--START--
   2.149 +....
   2.150 +....
   2.151 +[Sat Aug 29 08:56:03.065427 2020] [proxy_http:error] [pid 11:tid 140321722160896] [client 129.215.175.97:35390] AH01114: HTTP: failed to make connection to backend: araybwyn
   2.152 +[Sat Aug 29 08:56:03.065448 2020] [proxy_http:error] [pid 11:tid 140321730553600] [client 129.215.175.97:35392] AH01114: HTTP: failed to make connection to backend: araybwyn
   2.153 +[Sat Aug 29 08:56:03.068289 2020] [proxy:error] [pid 11:tid 140321480505088] (113)No route to host: AH00957: HTTP: attempt to connect to 192.168.202.8:8080 (*) failed
   2.154 +[Sat Aug 29 08:56:03.068291 2020] [proxy:error] [pid 11:tid 140321472112384] (113)No route to host: AH00957: HTTP: attempt to connect to 192.168.202.8:8080 (*) failed
   2.155 +[Sat Aug 29 08:56:03.068323 2020] [proxy_http:error] [pid 11:tid 140321480505088] [client 129.215.175.97:35394] AH01114: HTTP: failed to make connection to backend: araybwyn
   2.156 +[Sat Aug 29 08:56:03.068341 2020] [proxy_http:error] [pid 11:tid 140321472112384] [client 129.215.175.97:35396] AH01114: HTTP: failed to make connection to backend: araybwyn
   2.157 +--END--
   2.158 +
   2.159 +    #
   2.160 +    # These entries are from a couple of days ago.
   2.161 +    # Nothing recent.
   2.162 +    #
   2.163 +
   2.164 +        tail -f -n 1000 /var/log/httpd/tap.roe.ac.uk-access.log
   2.165 +
   2.166 +
   2.167 +--START--
   2.168 +....
   2.169 +....
   2.170 +129.215.175.97 - - [31/Aug/2020:15:48:19 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.171 +129.215.175.97 - - [31/Aug/2020:15:48:19 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.172 +129.215.175.97 - - [31/Aug/2020:15:48:19 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.173 +129.215.175.97 - - [31/Aug/2020:15:48:20 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.174 +129.215.175.97 - - [31/Aug/2020:15:48:20 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.175 +129.215.175.97 - - [31/Aug/2020:15:48:20 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.176 +129.215.175.97 - - [31/Aug/2020:15:48:20 +0000] "GET /firethorn/system/info HTTP/1.1" 200 1029 "-" "Python-urllib/3.6"
   2.177 +--END--
   2.178 +
   2.179 +    #
   2.180 +    # OK - that shows requests comming in, but not producing any error in the logs.
   2.181 +    #
   2.182 +
   2.183 +    #
   2.184 +    # Stelios noticed that all the requests are 'system/info' requests from the built in healthcheck.
   2.185 +    # None of our tests are showing up in the logs ..
   2.186 +    #
   2.187 +
   2.188 +        tail -f -n 1000 /var/log/httpd/error_log
   2.189 +
   2.190 +--START--
   2.191 +....
   2.192 +....
   2.193 +[Mon Aug 31 14:47:21.980356 2020] [lbmethod_heartbeat:notice] [pid 1:tid 140350772746496] AH02282: No slotmem from mod_heartmonitor
   2.194 +[Mon Aug 31 14:47:21.980942 2020] [http2:warn] [pid 1:tid 140350772746496] AH02951: mod_ssl does not seem to be enabled
   2.195 +[Mon Aug 31 14:47:21.981578 2020] [mpm_event:notice] [pid 1:tid 140350772746496] AH00489: Apache/2.4.34 (Fedora) configured -- resuming normal operations
   2.196 +[Mon Aug 31 14:47:21.981598 2020] [core:notice] [pid 1:tid 140350772746496] AH00094: Command line: 'httpd -D FOREGROUND'
   2.197 +[Mon Aug 31 15:14:49.708567 2020] [mpm_event:notice] [pid 1:tid 140350772746496] AH00491: caught SIGTERM, shutting down
   2.198 +[Mon Aug 31 15:24:50.461355 2020] [suexec:notice] [pid 1:tid 140365212715264] AH01232: suEXEC mechanism enabled (wrapper: /usr/sbin/suexec)
   2.199 +[Mon Aug 31 15:24:50.480608 2020] [lbmethod_heartbeat:notice] [pid 1:tid 140365212715264] AH02282: No slotmem from mod_heartmonitor
   2.200 +[Mon Aug 31 15:24:50.481352 2020] [http2:warn] [pid 1:tid 140365212715264] AH02951: mod_ssl does not seem to be enabled
   2.201 +[Mon Aug 31 15:24:50.484539 2020] [mpm_event:notice] [pid 1:tid 140365212715264] AH00489: Apache/2.4.34 (Fedora) configured -- resuming normal operations
   2.202 +[Mon Aug 31 15:24:50.484569 2020] [core:notice] [pid 1:tid 140365212715264] AH00094: Command line: 'httpd -D FOREGROUND'
   2.203 +--END--
   2.204 +
   2.205 +    #
   2.206 +    # Quite a few restarts in the main server log ?
   2.207 +    #
   2.208 +
   2.209 +
   2.210 +
   2.211 +
   2.212 +
   2.213 +# -----------------------------------------------------
   2.214 +# Try a request from outside.
   2.215 +#[user@desktop]
   2.216 +
   2.217 +    curl --head http://129.215.175.100:80
   2.218 +
   2.219 +--START--
   2.220 +HANGS
   2.221 +--END--
   2.222 +
   2.223 +
   2.224 +# -----------------------------------------------------
   2.225 +# Try a request from the physical host.
   2.226 +#[user@trop01]
   2.227 +
   2.228 +    curl --head http://129.215.175.100:80
   2.229 +
   2.230 +--START--
   2.231 +HANGS
   2.232 +--END--
   2.233 +
   2.234 +    Is that the right external address ?
   2.235 +
   2.236 +    curl --head http://tap.roe.ac.uk:80/
   2.237 +
   2.238 +# -----------------------------------------------------
   2.239 +# I suspect VM networking is broken ?
   2.240 +#[user@trop02]
   2.241 +
   2.242 +    source libvirt.settings
   2.243 +    virsh \
   2.244 +        -c $libvirtcon \
   2.245 +        list
   2.246 +
   2.247 +--START--
   2.248 + Id    Name                           State
   2.249 +----------------------------------------------------
   2.250 + 2     Acilamwen                      running
   2.251 + 3     Araybwyn                       running
   2.252 + 6     Ulov                           running
   2.253 +--END--
   2.254 +
   2.255 +
   2.256 +# -----------------------------------------------------
   2.257 +# Check the network connections assigned to the Apache VM.
   2.258 +#[user@trop02]
   2.259 +
   2.260 +   virsh \
   2.261 +        --connect ${libvirtcon:?} \
   2.262 +       dumpxml \
   2.263 +            Acilamwen \
   2.264 +   | xmlstarlet \
   2.265 +       select \
   2.266 +           --root \
   2.267 +           --indent \
   2.268 +           --template \
   2.269 +               --copy-of '//interface'
   2.270 +
   2.271 +--START--
   2.272 +<xsl-select>
   2.273 +  <interface type="network">
   2.274 +      <mac address="52:54:00:02:02:0f"/>
   2.275 +      <source network="default"/>
   2.276 +      <target dev="vnet0"/>
   2.277 +      <model type="virtio"/>
   2.278 +      <alias name="net0"/>
   2.279 +      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0"/>
   2.280 +    </interface>
   2.281 +  <interface type="bridge">
   2.282 +      <mac address="52:54:00:00:01:01"/>
   2.283 +      <source bridge="br0"/>
   2.284 +      <target dev="vnet1"/>
   2.285 +      <model type="virtio"/>
   2.286 +      <alias name="net1"/>
   2.287 +      <address type="pci" domain="0x0000" bus="0x00" slot="0x07" function="0x0"/>
   2.288 +    </interface>
   2.289 +  <interface type="bridge">
   2.290 +      <mac address="52:54:00:00:01:00"/>
   2.291 +      <source bridge="br0"/>
   2.292 +      <target dev="vnet2"/>
   2.293 +      <model type="virtio"/>
   2.294 +      <alias name="net2"/>
   2.295 +      <address type="pci" domain="0x0000" bus="0x00" slot="0x08" function="0x0"/>
   2.296 +    </interface>
   2.297 +</xsl-select>
   2.298 +--END--
   2.299 +
   2.300 +    #
   2.301 +    # Both net1 and net2 are connected to br0 ?
   2.302 +    #
   2.303 +
   2.304 +# -----------------------------------------------------
   2.305 +# Check the network interfaces inside the Apache VM.
   2.306 +#[user@trop02]
   2.307 +
   2.308 +    ssh Acilamwen '
   2.309 +        /sbin/ifconfig -a
   2.310 +        '
   2.311 +
   2.312 +--START--
   2.313 +ens3: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
   2.314 +        inet 192.168.202.15  netmask 255.255.255.0  broadcast 192.168.202.255
   2.315 +        inet6 fe80::5054:ff:fe02:20f  prefixlen 64  scopeid 0x20<link>
   2.316 +        ether 52:54:00:02:02:0f  txqueuelen 1000  (Ethernet)
   2.317 +        RX packets 461819  bytes 119429633 (113.8 MiB)
   2.318 +        RX errors 0  dropped 0  overruns 0  frame 0
   2.319 +        TX packets 376525  bytes 56700876 (54.0 MiB)
   2.320 +        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
   2.321 +
   2.322 +ens5: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
   2.323 +        inet 129.215.175.101  netmask 255.255.255.0  broadcast 129.215.175.255
   2.324 +        inet6 fe80::5054:ff:fe00:101  prefixlen 64  scopeid 0x20<link>
   2.325 +        ether 52:54:00:00:01:01  txqueuelen 1000  (Ethernet)
   2.326 +        RX packets 2964555  bytes 188649317 (179.9 MiB)
   2.327 +        RX errors 0  dropped 334  overruns 0  frame 0
   2.328 +        TX packets 268573  bytes 94658626 (90.2 MiB)
   2.329 +        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
   2.330 +
   2.331 +ens7: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
   2.332 +        inet 129.215.175.100  netmask 255.255.255.0  broadcast 129.215.175.255
   2.333 +        inet6 fe80::5054:ff:fe00:100  prefixlen 64  scopeid 0x20<link>
   2.334 +        ether 52:54:00:00:01:00  txqueuelen 1000  (Ethernet)
   2.335 +        RX packets 2696547  bytes 163355072 (155.7 MiB)
   2.336 +        RX errors 0  dropped 334  overruns 0  frame 0
   2.337 +        TX packets 1790  bytes 125556 (122.6 KiB)
   2.338 +        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
   2.339 +--END--
   2.340 +
   2.341 +# -----------------------------------------------------
   2.342 +# Check the default route.
   2.343 +#[user@trop02]
   2.344 +
   2.345 +    ssh Acilamwen '
   2.346 +        /usr/sbin/ip route
   2.347 +        '
   2.348 +
   2.349 +--START--
   2.350 +default via 129.215.175.126 dev ens7 proto static metric 101
   2.351 +129.215.175.0/24 dev ens5 proto kernel scope link src 129.215.175.101 metric 100
   2.352 +129.215.175.0/24 dev ens7 proto kernel scope link src 129.215.175.100 metric 101
   2.353 +172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
   2.354 +192.168.202.0/24 dev ens3 proto kernel scope link src 192.168.202.15 metric 102
   2.355 +--END--
   2.356 +
   2.357 +    #
   2.358 +    # I suspect that having 2 routes to the same network, 129.215.175.0/24, might be causing issues.
   2.359 +    # Because of the weightings, it will always prefer the 175.101 interface.
   2.360 +    #
   2.361 +
   2.362 +# -----------------------------------------------------
   2.363 +# Check a ssh connect.
   2.364 +#[user@trop02]
   2.365 +
   2.366 +    ssh Acilamwen '
   2.367 +        date
   2.368 +        hostname
   2.369 +        ssh Araybwyn "
   2.370 +            date
   2.371 +            hostname
   2.372 +            "
   2.373 +        date
   2.374 +        hostname
   2.375 +        '
   2.376 +
   2.377 +--START--
   2.378 +Mon 31 Aug 17:25:33 BST 2020
   2.379 +Acilamwen
   2.380 +Mon 31 Aug 17:31:17 BST 2020
   2.381 +Araybwyn
   2.382 +Mon 31 Aug 17:25:34 BST 2020
   2.383 +Acilamwen
   2.384 +--END--
   2.385 +
   2.386 +    #
   2.387 +    # Didn't quite work out the box, ssh keys needed to be accepted.
   2.388 +    # Worked after host keys accepted.
   2.389 +    #
   2.390 +
   2.391 +
   2.392 +# -----------------------------------------------------
   2.393 +# Start with Apache proxy.
   2.394 +#[user@trop02]
   2.395 +
   2.396 +    ssh Acilamwen
   2.397 +
   2.398 +# -----------------------------------------------------
   2.399 +# List the Docker containers.
   2.400 +#[user@trop02]
   2.401 +#[Stevedore@Acilamwen]
   2.402 +
   2.403 +        docker ps -a
   2.404 +
   2.405 +--START--
   2.406 +CONTAINER ID        IMAGE                     COMMAND                  CREATED             STATUS                         PORTS                NAMES
   2.407 +fc403b749771        firethorn/apache:2.1.25   "/usr/local/bin/http…"   About an hour ago   Exited (0) About an hour ago                        apache2
   2.408 +bc8d6979e451        firethorn/apache:2.1.25   "/usr/local/bin/http…"   24 months ago       Up About an hour               0.0.0.0:80->80/tcp   apache
   2.409 +--END--
   2.410 +
   2.411 +
   2.412 +# -----------------------------------------------------
   2.413 +# Login to the Apache container.
   2.414 +#[Stevedore@Acilamwen]
   2.415 +
   2.416 +    docker exec -it apache /bin/bash
   2.417 +
   2.418 +# -----------------------------------------------------
   2.419 +# Check the Apache config.
   2.420 +#[user@trop02]
   2.421 +#[Stevedore@Acilamwen]
   2.422 +#[root@apache]
   2.423 +
   2.424 +    cat /etc/httpd/conf.d/tap.roe.ac.uk.conf
   2.425 +
   2.426 +--START--
   2.427 +<VirtualHost *:80>
   2.428 +
   2.429 +    ServerName  tap.roe.ac.uk
   2.430 +    ServerAlias tap.metagrid.xyz
   2.431 +    ServerAdmin tap-admin@metagrid.xyz
   2.432 +
   2.433 +    ErrorLog  logs/tap.roe.ac.uk-error.log
   2.434 +    CustomLog logs/tap.roe.ac.uk-access.log combined
   2.435 +
   2.436 +    ProxyRequests Off
   2.437 +    ProxyPreserveHost On
   2.438 +
   2.439 +
   2.440 +    ProxyTimeout 6000
   2.441 +    Timeout 6000
   2.442 +
   2.443 +    <Proxy *>
   2.444 +        Order deny,allow
   2.445 +        Deny from 133.40.215.
   2.446 +        Allow from all
   2.447 +    </Proxy>
   2.448 +
   2.449 +    ## PROXY ENTRIES HERE ##
   2.450 +    #ProxyPassMatch    ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.451 +    #ProxyPassReverse  ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.452 +
   2.453 +    ## PROXY ENTRIES HERE ##
   2.454 +
   2.455 +    ProxyPassMatch    ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.456 +    ProxyPassReverse  ^/osa\/(.*)$  http://araybwyn:8080/firethorn/tap/54/$1
   2.457 +
   2.458 +    ProxyPassMatch    ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   2.459 +    ProxyPassReverse  ^/ssa\/(.*)$  http://araybwyn:8080/firethorn/tap/57/$1
   2.460 +
   2.461 +    ProxyPassMatch    ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   2.462 +    ProxyPassReverse  ^/vsa\/(.*)$  http://araybwyn:8080/firethorn/tap/60/$1
   2.463 +
   2.464 +    ProxyPassMatch    ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   2.465 +    ProxyPassReverse  ^/wsa\/(.*)$  http://araybwyn:8080/firethorn/tap/63/$1
   2.466 +
   2.467 +    ProxyPassMatch    ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   2.468 +    ProxyPassReverse  ^/firethorn\/(.*)$  http://araybwyn:8080/firethorn/$1
   2.469 +
   2.470 +</VirtualHost>
   2.471 +--END--
   2.472 +
   2.473 +
   2.474 +# -----------------------------------------------------
   2.475 +# Check we can reach the Firethorn service from here.
   2.476 +#[user@trop02]
   2.477 +#[Stevedore@Acilamwen]
   2.478 +#[root@apache]
   2.479 +
   2.480 +    curl --head http://araybwyn:8080/firethorn/system/info
   2.481 +
   2.482 +
   2.483 +--START--
   2.484 +HTTP/1.1 200
   2.485 +X-Clacks-Overhead: GNU Terry Pratchett
   2.486 +firethorn.auth.identity: http://araybwyn:8080/firethorn/community-member/817155
   2.487 +firethorn.auth.username: anon-VUMZ5KZIUKB3WAAAAF2EK5EAMM
   2.488 +firethorn.auth.community: friends
   2.489 +Content-Type: application/json;charset=ISO-8859-1
   2.490 +Content-Language: en-US
   2.491 +Content-Length: 930
   2.492 +Date: Mon, 31 Aug 2020 16:58:17 GMT
   2.493 +--END--
   2.494 +
   2.495 +
   2.496 +# -----------------------------------------------------
   2.497 +# What is the default Apache config ?
   2.498 +#[user@trop02]
   2.499 +#[Stevedore@Acilamwen]
   2.500 +#[root@apache]
   2.501 +
   2.502 +    pushd /etc/httpd/conf.d
   2.503 +
   2.504 +    ls -1 .
   2.505 +
   2.506 +--START--
   2.507 +README
   2.508 +autoindex.conf
   2.509 +genius.roe.ac.uk.conf
   2.510 +global.conf
   2.511 +osa.metagrid.xyz.conf
   2.512 +proxy-template
   2.513 +tap.roe.ac.uk.conf
   2.514 +userdir.conf
   2.515 +welcome.conf
   2.516 +--END--
   2.517 +
   2.518 +
   2.519 +    cat genius.roe.ac.uk.conf
   2.520 +
   2.521 +--START--
   2.522 +....
   2.523 +<VirtualHost *:80>
   2.524 +    ServerName  genius.roe.ac.uk
   2.525 +....
   2.526 +--END--
   2.527 +
   2.528 +
   2.529 +    cat osa.metagrid.xyz.conf
   2.530 +
   2.531 +--START--
   2.532 +....
   2.533 +<VirtualHost *:80>
   2.534 +    ServerName  osa.metagrid.xyz
   2.535 +....
   2.536 +--END--
   2.537 +
   2.538 +
   2.539 +    cat tap.roe.ac.uk.conf
   2.540 +
   2.541 +--START--
   2.542 +....
   2.543 +<VirtualHost *:80>
   2.544 +    ServerName  tap.roe.ac.uk
   2.545 +....
   2.546 +--END--
   2.547 +
   2.548 +    #
   2.549 +    # Config rules are - no matching host, use the first it finds.
   2.550 +    # Alphabetically, that would be the genius config.
   2.551 +    # Deleted the spare config files and we no longer get the 503 errors.
   2.552 +    #
   2.553 +
   2.554 +    #
   2.555 +    # Either works fine, from within the same physical trop host.
   2.556 +    # Or connection from anywhere else just hangs.
   2.557 +    #
   2.558 +
   2.559 +    #
   2.560 +    # Found this in previous notes :
   2.561 +
   2.562 +#---------------------------------------------------------------------
   2.563 +# Prevent bridge traffic being filtered.
   2.564 +# Haven't figured out how to make this persitent yet.
   2.565 +#[root@trop]
   2.566 +
   2.567 +    sysctl -w net.bridge.bridge-nf-call-arptables=0
   2.568 +    sysctl -w net.bridge.bridge-nf-call-iptables=0
   2.569 +    sysctl -w net.bridge.bridge-nf-call-ip6tables=0
   2.570 +
   2.571 +    #
   2.572 +    # Applied to trop02 and requests now work :-)
   2.573 +    #
   2.574 +
   2.575 +    Fixed BUT still has unsolved issues.
   2.576 +    1) Need to make the bridge-nf-call-iptables persistent on trop02.
   2.577 +    2) Why has Acilamwen got two of the floating IP addresses ?
   2.578 +    3) If both floating IP interfaces are enabled, does this mess up the default route ?
   2.579 +
   2.580 +
   2.581 +
   2.582 +