firethorn

changeset 4223:68d995c1fec3

Notes on merging
author Stelios <stv@roe.ac.uk>
date Thu May 09 19:49:00 2019 +0300 (2019-05-09)
parents ee3646af04b4
children 1e7585468a10
files doc/notes/stv/20190509-hg-merge.txt
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/doc/notes/stv/20190509-hg-merge.txt	Thu May 09 19:49:00 2019 +0300
     1.3 @@ -0,0 +1,336 @@
     1.4 +#
     1.5 +# <meta:header>
     1.6 +#   <meta:licence>
     1.7 +#     Copyright (c) 2018, ROE (http://www.roe.ac.uk/)
     1.8 +#
     1.9 +#     This information is free software: you can redistribute it and/or modify
    1.10 +#     it under the terms of the GNU General Public License as published by
    1.11 +#     the Free Software Foundation, either version 3 of the License, or
    1.12 +#     (at your option) any later version.
    1.13 +#
    1.14 +#     This information is distributed in the hope that it will be useful,
    1.15 +#     but WITHOUT ANY WARRANTY; without even the implied warranty of
    1.16 +#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    1.17 +#     GNU General Public License for more details.
    1.18 +#
    1.19 +#     You should have received a copy of the GNU General Public License
    1.20 +#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
    1.21 +#   </meta:licence>
    1.22 +# </meta:header>
    1.23 +#
    1.24 +#
    1.25 +
    1.26 +
    1.27 +## Notes are copied over from /zrq/
    1.28 +## Just Added comments where I had issues
    1.29 +
    1.30 +
    1.31 +# -----------------------------------------------------
    1.32 +# Create a new VM.
    1.33 +# Astoalith
    1.34 +
    1.35 +
    1.36 +# -----------------------------------------------------
    1.37 +# Login as Stevedore
    1.38 +#[user@trop]
    1.39 +
    1.40 +    ssh "${vmname:?}"
    1.41 +
    1.42 +# -----------------------------------------------------
    1.43 +# Install our secret function.
    1.44 +#[user@virtual]
    1.45 +
    1.46 +    mkdir "${HOME:?}/bin"
    1.47 +    vi "${HOME:?}/bin/secret"
    1.48 +
    1.49 +        ....
    1.50 +        ....
    1.51 +        ....
    1.52 +
    1.53 +    chmod u+x "${HOME:?}/bin/secret"
    1.54 +    secret 'frog'
    1.55 +
    1.56 +# -----------------------------------------------------
    1.57 +# Download our builder compose file
    1.58 +#[user@virtual]
    1.59 +
    1.60 +    wget -O builder.yml \
    1.61 +        http://wfau.metagrid.co.uk/code/firethorn/raw-file/tip/docker/compose/builder.yml
    1.62 +
    1.63 +# -----------------------------------------------------
    1.64 +# Set the target branch
    1.65 +#[user@virtual]
    1.66 +
    1.67 +    branch=2.1.23-zrq-update-depends
    1.68 +    branch=2.1.24-zrq-userdata-drop
    1.69 +    branch=2.1.25-zrq-tap-controller
    1.70 +
    1.71 +# -----------------------------------------------------
    1.72 +# Run our builder.
    1.73 +#[user@virtual]
    1.74 +
    1.75 +    export branch
    1.76 +    export secretsh=$(which secret)
    1.77 +
    1.78 +    docker-compose \
    1.79 +        --file "builder.yml" \
    1.80 +        run \
    1.81 +            builder
    1.82 +
    1.83 +    # -----------------------------------------------------
    1.84 +    # Test our secret function.
    1.85 +    #[root@builder]
    1.86 +
    1.87 +
    1.88 +[root@builder /]#         secret 'frog'
    1.89 +sed: can't read /root/secret.store: No such file or directory
    1.90 +
    1.91 +## Create /root/secret.store manually
    1.92 +
    1.93 +
    1.94 +    # -----------------------------------------------------
    1.95 +    # Initialise our paths.
    1.96 +    #[root@builder]
    1.97 +
    1.98 +        PATH=${PATH}:/builder/bin
    1.99 +
   1.100 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   1.101 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   1.102 +
   1.103 +        export FIRETHORN_HOME
   1.104 +        export FIRETHORN_CODE
   1.105 +
   1.106 +    # -----------------------------------------------------
   1.107 +    # Checkout a copy of our source code.
   1.108 +    #[root@builder]
   1.109 +
   1.110 +        02.01-checkout.sh
   1.111 +
   1.112 +
   1.113 +## Needed to fix ssh keys
   1.114 +## Add id_dsa
   1.115 +## Create config with entries for data, wfau (metagrid.co.uk)
   1.116 +## with PubkeyAcceptedKeyTypes=+ssh-dss
   1.117 +
   1.118 +
   1.119 +    # -----------------------------------------------------
   1.120 +    # Merge our branch into main.
   1.121 +    #[root@builder]
   1.122 +
   1.123 +        07.01-merge-start.sh
   1.124 +
   1.125 +    # -----------------------------------------------------
   1.126 +    # Update our version number.
   1.127 +    #[root@builder]
   1.128 +
   1.129 +        07.02-version-step.sh
   1.130 +
   1.131 +    # -----------------------------------------------------
   1.132 +    # Build our base images.
   1.133 +    #[root@builder]
   1.134 +
   1.135 +        04.01-buildbase.sh
   1.136 +
   1.137 +    # -----------------------------------------------------
   1.138 +    # Tag this version of builder as latest.
   1.139 +    #[root@builder]
   1.140 +
   1.141 +        source "${FIRETHORN_HOME}/merge.settings"
   1.142 +
   1.143 +        docker tag \
   1.144 +            "firethorn/builder:${newversion}" "firethorn/builder:latest"
   1.145 +
   1.146 +    # -----------------------------------------------------
   1.147 +    # Exit this version of builder.
   1.148 +    #[root@builder]
   1.149 +
   1.150 +        exit
   1.151 +
   1.152 +# -----------------------------------------------------
   1.153 +# Run a new copy of builder.
   1.154 +#[user@virtual]
   1.155 +
   1.156 +    export branch
   1.157 +    export secretsh=$(which secret)
   1.158 +
   1.159 +    docker-compose \
   1.160 +        --file "builder.yml" \
   1.161 +        run \
   1.162 +            builder
   1.163 +
   1.164 +    # -----------------------------------------------------
   1.165 +    # Test our secret function.
   1.166 +    #[root@builder]
   1.167 +
   1.168 +        secret 'frog'
   1.169 +
   1.170 +
   1.171 +[root@builder /]#         secret 'frog'
   1.172 +sed: can't read /root/secret.store: No such file or directory
   1.173 +
   1.174 +## Create /root/secret.store manually
   1.175 +
   1.176 +    # -----------------------------------------------------
   1.177 +    # Initialise our paths.
   1.178 +    #[root@builder]
   1.179 +
   1.180 +        PATH=${PATH}:/builder/bin
   1.181 +
   1.182 +        : ${FIRETHORN_HOME:=/etc/firethorn}
   1.183 +        : ${FIRETHORN_CODE:=/var/local/build/firethorn}
   1.184 +
   1.185 +        export FIRETHORN_HOME
   1.186 +        export FIRETHORN_CODE
   1.187 +
   1.188 +    # -----------------------------------------------------
   1.189 +    # Build our base images.
   1.190 +    #[root@builder]
   1.191 +
   1.192 +        04.01-buildbase.sh
   1.193 +
   1.194 +    # -----------------------------------------------------
   1.195 +    # Compile our Java code.
   1.196 +    #[root@builder]
   1.197 +
   1.198 +        05.01-javamaven.sh
   1.199 +
   1.200 +    # -----------------------------------------------------
   1.201 +    # Build our Java containers.
   1.202 +    #[root@builder]
   1.203 +
   1.204 +        05.02-javadocker.sh
   1.205 +
   1.206 +    # -----------------------------------------------------
   1.207 +    # Fetch our Python code.
   1.208 +    #[root@builder]
   1.209 +
   1.210 +        echo "Fetching client source"
   1.211 +
   1.212 +        ftpysrc='/var/local/build/client'
   1.213 +       #ftpygit='https://github.com/Zarquan/firethorn.py.git'
   1.214 +        ftpygit='https://github.com/stvoutsin/firethorn.py.git'
   1.215 +
   1.216 +        if [  -e "${ftpysrc:?}" ]
   1.217 +        then
   1.218 +            pushd "${ftpysrc:?}"
   1.219 +
   1.220 +                echo "Updating Python code"
   1.221 +                git pull
   1.222 +
   1.223 +            popd
   1.224 +        else
   1.225 +            pushd "$(dirname ${ftpysrc:?})"
   1.226 +
   1.227 +                echo "Cloning Python code from [${ftpygit:?}]"
   1.228 +                git clone "${ftpygit:?}" "$(basename ${ftpysrc:?})"
   1.229 +
   1.230 +            popd
   1.231 +        fi
   1.232 +
   1.233 +    # -----------------------------------------------------
   1.234 +    # Delete the embedded metadata.
   1.235 +    #[root@builder]
   1.236 +
   1.237 +        pushd "${ftpysrc:?}"
   1.238 +
   1.239 +            du -h  firethorn/meta
   1.240 +            rm -rf firethorn/meta
   1.241 +
   1.242 +        popd
   1.243 +
   1.244 +## du: cannot access 'firethorn/meta': No such file or directory
   1.245 +
   1.246 +
   1.247 +    # -----------------------------------------------------
   1.248 +    # Build our Python container.
   1.249 +    #[root@builder]
   1.250 +
   1.251 +        echo "Building client image"
   1.252 +
   1.253 +        pushd "${FIRETHORN_CODE:?}"
   1.254 +
   1.255 +            source 'bin/util.sh'
   1.256 +            export buildtag=$(getbuildtag)
   1.257 +            export buildsrc=$(pwd)
   1.258 +            export ftpysrc
   1.259 +
   1.260 +            docker-compose \
   1.261 +                --file docker/compose/client/firethorn-py.yml \
   1.262 +                build
   1.263 +
   1.264 +        popd
   1.265 +
   1.266 +# -----------------------------------------------------
   1.267 +# Run our Python tests, 20180712-03-merge-tests.txt
   1.268 +# -----------------------------------------------------
   1.269 +
   1.270 +    # -----------------------------------------------------
   1.271 +    # Update our Mercurial config.
   1.272 +    #[root@builder]
   1.273 +
   1.274 +        07.02-mercurial-user.sh
   1.275 +
   1.276 +## /builder/bin/07.02-mercurial-user.sh: line 31: gitrepo: parameter null or not set
   1.277 +
   1.278 +## Add values to secret.store
   1.279 +
   1.280 +
   1.281 +
   1.282 +
   1.283 +    # -----------------------------------------------------
   1.284 +    # Commit our merged code.
   1.285 +    #[root@builder]
   1.286 +
   1.287 +        07.03-merge-commit.sh
   1.288 +
   1.289 +    # -----------------------------------------------------
   1.290 +    # Push changes to Mercurial
   1.291 +    #[root@builder]
   1.292 +
   1.293 +        08.03-mercurial-push.sh
   1.294 +
   1.295 +## ssh missing
   1.296 +## remote: root@wfau.metagrid.co.uk: Permission denied (publickey,gssapi-keyex,gssapi-with-mic).
   1.297 +
   1.298 +## Had to manually copy paste code from this two scripts to get this working..
   1.299 +## Something missing, but no time to check that now
   1.300 +
   1.301 +
   1.302 +    # -----------------------------------------------------
   1.303 +    # Tag our new images
   1.304 +    #[root@builder]
   1.305 +
   1.306 +        08.01-docker-latest.sh
   1.307 +
   1.308 +    # -----------------------------------------------------
   1.309 +    # Push images to Docker.
   1.310 +    #[root@builder]
   1.311 +
   1.312 +        08.02-docker-push.sh
   1.313 +
   1.314 +    # -----------------------------------------------------
   1.315 +    # Push binaries to Maven repo.
   1.316 +    #[root@builder]
   1.317 +
   1.318 +        08.04-maven-push.sh
   1.319 +
   1.320 +..
   1.321 +
   1.322 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn-testing/.maven-metadata-local.xml.sha1.3FZcDy" failed: Permission denied (13)
   1.323 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn-webapp/.maven-metadata-local.xml.KYw94D" failed: Permission denied (13)
   1.324 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn-webapp/.maven-metadata-local.xml.md5.dNaexJ" failed: Permission denied (13)
   1.325 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn-webapp/.maven-metadata-local.xml.sha1.I5ViZO" failed: Permission denied (13)
   1.326 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn/.maven-metadata-local.xml.CyZ4rU" failed: Permission denied (13)
   1.327 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn/.maven-metadata-local.xml.md5.EgHRUZ" failed: Permission denied (13)
   1.328 +rsync: mkstemp "/var/local/websites/data/wfau/maven/firethorn/uk/ac/roe/wfau/firethorn/.maven-metadata-local.xml.sha1.6oxEn5" failed: Permission denied (13)
   1.329 +
   1.330 +..
   1.331 +
   1.332 +## rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1189) [sender=3.1.3]
   1.333 +
   1.334 +
   1.335 +
   1.336 +
   1.337 +
   1.338 +
   1.339 +