Commit 89d55e11 authored by Julie Brossard's avatar Julie Brossard
Browse files

Merge branch 'ci' into 'master'

Continuous Integration for WASP

See merge request !11
parents 5f73228a 60fac745
Pipeline #9535 failed
Data/**/*.tif filter=lfs diff=lfs merge=lfs -text
Data/**/*.jpg filter=lfs diff=lfs merge=lfs -text
variables:
BUILD_ENV_REGISTRY: $CI_REGISTRY/remote_modules/wasp
WASP_TEST: $CI_PROJECT_DIR/Data
stages:
- container
- build
- validation
- quality
# ----------------------------[ container ]----------------------------------
build-env-centos7:
stage: container
image:
name: gcr.io/kaniko-project/executor:debug-v0.16.0
entrypoint: [""]
tags:
- docker
before_script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}"
> /kaniko/.docker/config.json
script:
- /kaniko/executor --cleanup
--cache=true
--cache-repo=$CI_REGISTRY_IMAGE/wasp-env-centos7-cache
--verbosity warn
--dockerfile $CI_PROJECT_DIR/CI/RH7/Dockerfile
--context $CI_PROJECT_DIR/CI/RH7
--destination $CI_REGISTRY_IMAGE/wasp-env-centos7:$CI_COMMIT_REF_NAME
only:
refs:
- master
- develop
- release-*
- ci
changes:
- CI/RH7/*
#~ - .gitlab-ci.yml
build-env-ubuntu:
stage: container
image:
name: gcr.io/kaniko-project/executor:debug-v0.16.0
entrypoint: [""]
tags:
- docker
before_script:
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}"
> /kaniko/.docker/config.json
script:
- /kaniko/executor --cleanup
--cache=true
--cache-repo=$CI_REGISTRY_IMAGE/wasp-env-ubuntu-cache
--verbosity warn
--dockerfile $CI_PROJECT_DIR/CI/Ubuntu/Dockerfile
--context $CI_PROJECT_DIR/CI/Ubuntu
--destination $CI_REGISTRY_IMAGE/wasp-env-ubuntu:$CI_COMMIT_REF_NAME
only:
refs:
- master
- develop
- release-*
- ci
changes:
- CI/Ubuntu/*
#~ - .gitlab-ci.yml
# ----------------------------[ build ]----------------------------------
# - CentOS7
.build-centos7:
stage: build
tags:
- docker
script:
- echo $CI_COMMIT_SHA > sha1.txt
- mkdir build
- cd build
- cmake -GNinja
-DCMAKE_PREFIX_PATH=/opt/otb
-DCMAKE_INSTALL_PREFIX=../install
-DCMAKE_CXX_FLAGS=-Wl,--as-needed
-DBUILD_TESTING=ON
../
- ninja install
- rclone copy s3_otb:wasp-data/test_Preprocessing ../Data/test_Preprocessing
- rclone copy s3_otb:wasp-data/test_MuscateMetadata ../Data/test_MuscateMetadata
- ctest -E LUT
- mkdir ../pkg-build
- cd ../pkg-build
- cmake -GNinja
-DCMAKE_INSTALL_PREFIX=../install
-DWASP_PACKAGE_LABEL=$CI_COMMIT_SHORT_SHA
../Package
- ninja
artifacts:
paths:
- pkg-build/WASP-*.run
# use ad-hoc container for master/develop/release/ci
build-centos7:
extends: .build-centos7
image: $CI_REGISTRY_IMAGE/wasp-env-centos7:$CI_COMMIT_REF_NAME
only:
refs:
- master
- develop
- release-*
- ci
# use develop container for other branches
build-centos7-feat:
extends: .build-centos7
image: $CI_REGISTRY_IMAGE/wasp-env-centos7:develop
except:
refs:
- master
- develop
- release-*
- ci
# - Ubuntu
.build-ubuntu:
stage: build
tags:
- docker
script:
- echo $CI_COMMIT_SHA > sha1.txt
- mkdir build
- cd build
- cmake -GNinja
-DCMAKE_PREFIX_PATH=/opt/otb
-DCMAKE_CXX_FLAGS=--coverage
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-DBUILD_TESTING=ON
-DCMAKE_BUILD_TYPE=Debug
../
- ninja
- rclone copy s3_otb:wasp-data/test_Preprocessing ../Data/test_Preprocessing
- rclone copy s3_otb:wasp-data/test_MuscateMetadata ../Data/test_MuscateMetadata
- ctest -D ExperimentalTest -E LUT
- find . -name "*.gcda" -exec gcov -p '{}' > /dev/null \;
- ls *.gcov | grep -E -v '^\^#Code#' | xargs -L 1 rm
- echo Filtered $(ls *.gcov | wc -l) gcov reports
- gcovr -r $CI_PROJECT_DIR -x -g --object-directory=. > ../coverage_cpp.xml
- cd ..
- echo Generated coverage report with $(grep -o '<class ' coverage_cpp.xml | wc -l) classes
- saxon-xslt -o ctest_report.xml build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
CI/ctest2junit.xsl
- find Code -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
sort -u > header_directories
- cppcheck -j 8 -q --xml --xml-version=2 --enable=all
--language=c++ --std=c++11 --platform=unix64
--includes-file=header_directories Code
2> cppcheck_report.xml
artifacts:
paths:
- coverage_cpp.xml
- ctest_report.xml
- cppcheck_report.xml
# use ad-hoc container for master/develop/release/ci
build-ubuntu:
extends: .build-ubuntu
image: $CI_REGISTRY_IMAGE/wasp-env-ubuntu:$CI_COMMIT_REF_NAME
only:
refs:
- master
- develop
- release-*
- ci
# use develop container for other branches
build-ubuntu-feat:
extends: .build-ubuntu
image: $CI_REGISTRY_IMAGE/wasp-env-ubuntu:develop
except:
refs:
- master
- develop
- release-*
- ci
# ----------------------------[ validation ]----------------------------------
.valid-centos7:
stage: validation
image: $CI_REGISTRY_IMAGE/wasp-env-centos7:$CI_COMMIT_REF_NAME
tags:
- docker
needs: ["build-centos7"]
script:
- $(ls ./pkg-build/WASP-*.run | head -n 1) --target WASP
- export PYTHONPATH=$(pwd)/WASP/bin:$(pwd)/WASP/lib/python
- rclone copy s3_otb:wasp-data/test_S2B_T31TCH_1_20171008 Data/test_S2B_T31TCH_1_20171008
- rclone copy s3_otb:wasp-data/test_VNS_FR-LUS_4_20180207 Data/test_VNS_FR-LUS_4_20180207
- cd WASP
- nosetests --with-xunit --with-coverage --cover-xml --cover-package=bin lib/python/test_S2B_T31TCH_1_20171008.py lib/python/test_VNS_FR-LUS_4_20180207.py
- cd ..
- sed -i "s@<source>.*</source>@<source>$CI_PROJECT_DIR</source>@" WASP/coverage.xml
- sed -i 's@filename="@filename="PythonScripts/@' WASP/coverage.xml
- "pylint PythonScripts -r n --msg-template=\"{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}\" > pylint-report.txt || exit 0"
artifacts:
paths:
- WASP/*.xml
- pylint-report.txt
# use ad-hoc container for master/develop/release/ci
valid-centos7:
extends: .valid-centos7
image: $CI_REGISTRY_IMAGE/wasp-env-centos7:$CI_COMMIT_REF_NAME
only:
refs:
- master
- develop
- release-*
- ci
# use develop container for other branches
valid-centos7-feat:
extends: .valid-centos7
image: $CI_REGISTRY_IMAGE/wasp-env-centos7:develop
except:
refs:
- master
- develop
- release-*
- ci
# ----------------------------[ quality ]----------------------------------
quality-reports:
stage: quality
image: $CI_REGISTRY_IMAGE/wasp-env-ubuntu:develop
tags:
- docker
needs: ["valid-centos7", "build-ubuntu"]
script:
- sonar-scanner -Dsonar.login=$SONAR_TOKEN
only:
refs:
- develop
variables:
- $SONAR_TOKEN
# Continuous Integration for WASP
This document presents the CI implemented for WASP.
## Platforms tests
WASP is tested on 2 platforms (64bits):
* CentOS 7, with compiler GCC 7
* Ubuntu 20.04 LTS, with compiler GCC 9
These environments are defined as `Dockerfiles` located in `CI/RH7` and `CI/Ubuntu`.
## Test Data
The test data has been shrunk with the script `crop_s2.sh`. It fills a large part
of the input images with zeros (or the adequate no-data value), leaving only the
original data on a small strip. The output images are then stored in GeoTIFF with
DEFLATE compression to save space.
The test data is split between 2 places:
* [S3 bucket WASP-Data](https://s3.orfeo-toolbox.org/minio/wasp-data/) : contains
the input data for tests, the shrunk times series of S2 and Venus
* Git repository under `Data`: contains the baselines, stored as LFS objects.
## Pipelines
This is the overall structure of the CI pipeline:
```mermaid
graph LR;
subgraph "stage:Container"
A["CentOS container"];
B["Ubuntu container"];
end
subgraph "stage:Build"
C["CentOS build"];
D["Ubuntu build"];
end
A-->C
B-->D
subgraph "stage::Validation"
C-->E["CentOS validation"];
end
subgraph "stage:Quality"
E-->F["Quality report"];
end
D-->F
```
There are some specificites regarding the branches. Some jobs are not run for
every branch.
| Branch | Container | Build | Validation | Quality |
| ------ | -------- | ----- | ---------- | ------- |
| master | :white_check_mark: | :white_check_mark: | :white_check_mark: | |
| develop | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| release-\* | :white_check_mark: | :white_check_mark: | :white_check_mark: | |
| ci | :white_check_mark: | :white_check_mark: | :white_check_mark: | |
| *others* | | :white_check_mark: | :white_check_mark: | |
If the container is built on a given branch, the same container is used in
the following jobs of the pipeline. When no container is built in the pipeline,
the container from `develop` is used.
### Container
We use Kaniko to build base environments for the CI (CentOS 7 and Ubuntu 20.04).
The docker images are uploaded to the
[registry](https://gitlab.orfeo-toolbox.org/remote_modules/wasp/container_registry)
with the following names:
* `wasp-env-centos7`
* `wasp-env-ubuntu`
They are tagged with the branch name.
### Build
On CentOS 7, the steps are:
* Build WASP in `Release` mode
* Run C++ unit tests
* Create `.run` package
On Ubuntu, the steps are:
* Build WASP in `Debug` mode, with `--coverage` option
* Run C++ unit tests
* Generate C++ coverage report with `gcov` and `gcovr`
* Convert Test report from CTest to Junit format
* Run `cppcheck` for C++ code analysis
This job produces the reports (test, coverage, code analysis) for the C++ side
of WASP.
### Validation
Only run on CentOS 7:
* Install the `.run` package from build stage
* Run Python validation tests for the whole chain, with `nosetests` and coverage options
* Run `pylint` for Python code analysis
This job produces the reports (test, coverage, code analysis) for the Python side
of WASP.
### Quality
Run on the Ubuntu container that contains `sonar-scanner`:
* Send all quality reports to [SonarQube](https://sonar.orfeo-toolbox.org/dashboard?id=wasp).
## Future improvements
There are some points that can be improved:
* The C++ unit tests don't cover the applications in `WeightCalculation` and
`UpdateSynthesis`, so the coverage is not measured for these folders.
* The tests using directional LUT are disabled, they can be enabled when the
input data and baselines are uploaded.
* The Python tests run are `test_S2B_T31TCH_1_20171008` and `test_VNS_FR-LUS_4_20180207`.
Even with shrunk products, the whole chain is a bit long to run. Optimizations
should be performed before enabling the other Python tests. Ideally, the pipeline
should run in less than 30min.
FROM centos:7
LABEL VERSION="1.0" ARCHITECTURE="amd64"
LABEL MAINTAINER "WASP development team"
ADD yum.conf /etc/yum.conf
RUN yum -y update; yum clean all
RUN yum -y install epel-release centos-release-scl
RUN yum -y install \
bzip2 \
devtoolset-7 \
doxygen \
file \
libffi-devel \
libXcursor libXcursor-devel \
libXi libXi-devel \
libXinerama libXinerama-devel \
libXrandr libXrandr-devel \
mesa-libGL mesa-libGL-devel \
mesa-libGLU mesa-libGLU-devel \
ninja-build \
patch \
patchelf \
rclone \
rh-git218 \
rh-python36 \
wget \
&& yum clean all
# Install cmake 3.14
RUN cd /tmp \
&& wget -nv https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz \
&& tar -xzf cmake-3.14.0-Linux-x86_64.tar.gz \
&& cp -R cmake-3.14.0-Linux-x86_64/* /usr/local/ \
&& rm -rf cmake-3.14.0-Linux-x86_64*
# ADD rclone.conf /root/.config/rclone/
ENV IMAGE_NAME=rh7
SHELL ["/usr/bin/scl", "enable", "devtoolset-7", "rh-git218", "rh-python36"]
# Install python deps
RUN python3 -m pip install \
coverage \
lxml \
nose \
numpy \
pylint
# Install OTB
RUN cd /opt \
&& wget -nv https://www.orfeo-toolbox.org/packages/archives/OTB/OTB-7.1.0-Linux64.run \
&& chmod +x OTB-7.1.0-Linux64.run \
&& ./OTB-7.1.0-Linux64.run --target /opt/otb \
&& source /opt/otb/otbenv.profile \
&& ctest -S /opt/otb/share/otb/swig/build_wrapping.cmake -VV \
&& rm OTB-7.1.0-Linux64.run
ENV LD_LIBRARY_PATH=/opt/otb/lib \
OTB_APPLICATION_PATH=/opt/otb/lib/otb/applications \
PATH=/opt/otb/bin/:$PATH \
PYTHONPATH=/opt/otb/lib/python \
GDAL_DATA=/opt/otb/share/gdal \
PROJ_LIB=/opt/otb/share/proj \
GEOTIFF_CSV=/opt/otb/share/epsg_csv
ADD rclone.conf /root/.config/rclone/
RUN cd /opt \
&& wget -nv https://github.com/megastep/makeself/releases/download/release-2.4.2/makeself-2.4.2.run \
&& chmod +x makeself-2.4.2.run \
&& ./makeself-2.4.2.run --target makeself \
&& cp makeself/*.sh /usr/local/bin \
&& rm -r makeself \
&& rm makeself-2.4.2.run
ENTRYPOINT ["scl", "enable", "devtoolset-7", "rh-git218", "rh-python36", "--"]
[s3_otb]
type = s3
provider = Minio
env_auth = true
endpoint = https://s3.orfeo-toolbox.org
[main]
cachedir=/var/cache/yum/$basearch/$releasever
keepcache=0
debuglevel=2
logfile=/var/log/yum.log
exactarch=0
obsoletes=0
gpgcheck=0
plugins=0
installonly_limit=5
exclude=*.i?86 *.i686
# ----------------------------------------------------------------------------
# Multi-stage build: Stage #1 (CppCheck build)
# ----------------------------------------------------------------------------
FROM ubuntu:20.04 as builder
ENV DEBIAN_FRONTEND noninteractive
ENV CPPCHECK_VERSION 1.87
RUN echo "Europe/Paris" > /etc/timezone
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
curl \
ca-certificates \
tar \
g++ \
gcc \
gzip \
libpcre3-dev \
make \
python \
&& rm -rf /var/lib/apt/lists/*
# Install up-to-date CppCheck version
RUN cd /tmp \
&& curl -L -o cppcheck-${CPPCHECK_VERSION}.tar.gz \
https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz \
&& tar xzf cppcheck-${CPPCHECK_VERSION}.tar.gz \
&& cd cppcheck-${CPPCHECK_VERSION} \
&& make SRCDIR=build CFGDIR=/usr/share/cppcheck/cfg HAVE_RULES=yes \
CXXFLAGS="-O2 -DNDEBUG -Wall -Wno-sign-compare -Wno-unused-function"
# ----------------------------------------------------------------------------
# Multi-stage build: Stage #2 (main build)
# ----------------------------------------------------------------------------
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND noninteractive
ENV CPPCHECK_VERSION 1.87
ENV SONAR_SCANNER_VERSION 3.3.0.1492
RUN echo "Europe/Paris" > /etc/timezone
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
bzip2 \
ca-certificates \
ccache \
cmake \
curl \
doxygen \
file \
g++ \
gcc \
gcovr \
git \
libglvnd-dev \
libglu1-mesa-dev \
libpcre3 \
libpython3-dev \
libsaxon-java \
libtool \
make \
ninja-build \
openjdk-8-jre-headless \
patch \
patchelf \
python3 \
python3-pip \
rclone \
unzip \
wget \
&& rm -rf /var/lib/apt/lists/*
# Install python deps
RUN python3 -m pip install \
coverage \
lxml \
nose \
numpy \
pylint
ADD rclone.conf /root/.config/rclone/
ADD itk_gcc_check.diff /opt
ENV IMAGE_NAME=focal
COPY --from=builder /tmp/cppcheck-${CPPCHECK_VERSION}/cppcheck /usr/bin/
COPY --from=builder /tmp/cppcheck-${CPPCHECK_VERSION}/cfg /usr/share/cppcheck/cfg
COPY --from=builder /tmp/cppcheck-${CPPCHECK_VERSION}/addons/*.py /usr/bin/
COPY --from=builder /tmp/cppcheck-${CPPCHECK_VERSION}/addons/*/*.py /usr/bin/
# NB: Zip archive name is sonar-scanner-cli-x.y.z.nnnn-linux.zip
# but created dir is sonar-scanner-x.y.z.nnnn-linux (i.e. without "-cli" keyword)
RUN cd /usr/local \
&& curl -L -o ./sonar-scanner.zip \
https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip \
&& unzip sonar-scanner.zip \
&& rm sonar-scanner.zip \
&& mv sonar-scanner-${SONAR_SCANNER_VERSION}-linux sonar-scanner
ENV SONAR_RUNNER_HOME=/usr/local/sonar-scanner
SHELL ["/bin/bash", "-c"]
# Install OTB
RUN cd /opt \
&& wget -nv https://www.orfeo-toolbox.org/packages/archives/OTB/OTB-7.1.0-Linux64.run \
&& chmod +x OTB-7.1.0-Linux64.run \
&& ./OTB-7.1.0-Linux64.run --target /opt/otb \
&& patch -p1 < itk_gcc_check.diff \
&& source /opt/otb/otbenv.profile \
&& ctest -S /opt/otb/share/otb/swig/build_wrapping.cmake -VV \
&& rm OTB-7.1.0-Linux64.run
ENV LD_LIBRARY_PATH=/opt/otb/lib \
OTB_APPLICATION_PATH=/opt/otb/lib/otb/applications \
PATH=$PATH:/opt/otb/bin:/usr/local/sonar-scanner/bin \
PYTHONPATH=/opt/otb/lib/python \
GDAL_DATA=/opt/otb/share/gdal \
PROJ_LIB=/opt/otb/share/proj \
GEOTIFF_CSV=/opt/otb/share/epsg_csv
# ENTRYPOINT ["/bin/bash", "-c"]
--- src-orig/otb/include/ITK-4.13/vcl_compiler.h 2020-11-27 12:08:48.419844576 +0100
+++ src/otb/include/ITK-4.13/vcl_compiler.h 2020-11-27 12:09:26.856330698 +0100
@@ -98,7 +98,7 @@
# define VCL_GCC_80
# endif
# else
-# error "Dunno about this gcc"
+# define VCL_GCC_RECENT
# endif
#endif
[s3_otb]
type = s3
provider = Minio
env_auth = true
endpoint = https://s3.orfeo-toolbox.org
#!/bin/bash
if [ $# -lt 2 ]; then
echo "Usage : $0 input_dir output_dir"
exit 1
fi
dump_md()
{
gdalinfo $1 > dump.txt
}
get_pixel_type()
{
gdalType=$(cat dump.txt | grep '^Band 1 ' | grep -oE 'Type=[^,]+' | cut -d '=' -f 2)
case "$gdalType" in