-
Tristan Laurent authoredTristan Laurent authored
.gitlab-ci.yml 13.59 KiB
#
# Copyright (C) 2005-2024 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
# We have to use directly git-lfs wrapping commands to have an
# efficient cloning step.
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)
variables:
BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
DOCKER_DRIVER: overlay2
GIT_DEPTH: "3"
# Disable automatic checkout to let us fetch LFS before
GIT_CHECKOUT: "false"
# The fetch strategy fails with LFS and GitLab
GIT_STRATEGY: "clone"
PACKAGE_OTB_VERSION: "latest"
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
when: never
- if: '$CI_COMMIT_BRANCH'
stages:
- precheck
- prepare
- build
- report
- deploy
- docker
.general:
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
.common:
extends: .general
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
after_script: - python3 -u CI/cdash_handler.py
.common-build:
extends: .common
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: build
artifacts:
when: always
expire_in: 24 hrs
paths:
- build/*/*.log #CMake log
- log/*.txt # Others
- CookBook-*-html.tar.gz
- CookBook-*.pdf
- OTB-Doxygen-*.tar.bz2
- OTB-*.tar.gz
- OTB-*.zip
- build/compile_commands.json
- build/ctest_report.xml
- build/cppcheck_report.xml
- build/coverage_report.xml
- build/Testing/Temporary/*.txt #DEBUG
- Docker/Dockerfile_CI
- Docker/system-dependencies.txt
.common-prepare:
extends: .general
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: prepare
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
artifacts:
expire_in: 24 hrs
when: always
paths:
- sb_branch.txt # Needed to checkout correct branch in build step
- build/*/*/*/*.log # Superbuild log
- build/*/*/*/*.cmake
#-------------------------- precheck job ---------------------------------------
fast-build:
extends: .common
only:
- merge_requests
- branches
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:22.04
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-22.04-fast
- ccache -s
legal-check:
extends: .common
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7 variables:
GIT_DEPTH: ""
allow_failure: true
script:
- ./CI/contributors_check.sh
- ./CI/headers_check.py
after_script: []
#------------------------- prepare & build jobs --------------------------------
## Ubuntu superbuild
ubuntu-22-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:22.04
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=ubuntu-22.04-llvm-full -DOTB_FULL_BUILD:BOOL=ON
ubuntu-24-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:24.04
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=ubuntu-24.04-llvm-xdk-full -DOTB_FULL_BUILD:BOOL=ON
artifacts:
untracked: false
when: on_success
access: all
expire_in: 24hrs
paths:
- xdk/lib/python3/*
ubuntu-22-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:22.04
script:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-22.04-llvm-xdk-full
- mv build/CookBook-*-html.tar.gz . || true
- mv build/Documentation/Cookbook/latex/CookBook-*.pdf . || true
- mv build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 . || true
- ctest -V -S CI/superbuild_packaging.cmake -DIMAGE_NAME:string=ubuntu-22.04-llvm-xdk-full -DPACKAGE_BY_MODULE:BOOL=ON
- ctest -V -S CI/superbuild_packaging.cmake -DIMAGE_NAME:string=ubuntu-22.04-llvm-xdk-full -DPACKAGE_BY_MODULE:BOOL=OFF
- mv build_packages/OTB-*.tar.gz . || true
needs:
- job: ubuntu-22-prepare
artifacts: true
# Ubuntu 24 build needed for gdal bindings with python 3.12. We package the compiled bindings for python 3.12, this package will be downloaded and extracted automatically when
# a user lauches the otbenv profile after unpacking the official archive for the first time (which contains bindings for ubuntu 20 and python 3.8 initially)
python3.12-gdal-bindings:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:24.04
only:
changes:
- SuperBuild/CMake/External_gdal.cmake
refs:
- develop
- /^release-[0-9]+\.[0-9]+$/
- merge_requests
script:
- |
MAJOR_VERSION=$(egrep "set\(OTB_VERSION_MAJOR" < CMakeLists.txt | egrep -o "[0-9]+")
MINOR_VERSION=$(egrep "set\(OTB_VERSION_MINOR" < CMakeLists.txt | egrep -o "[0-9]+")
PATCH_VERSION=$(egrep "set\(OTB_VERSION_PATCH" < CMakeLists.txt | egrep -o "[0-9]+")
PACKAGE_OTB_VERSION="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION"
- find xdk/lib/python3/dist-packages/osgeo/ -type f -name "*.so" -printf "%P\n" | tar -czf OTB-$PACKAGE_OTB_VERSION-GDAL-bindings-py312.tar.gz --no-recursion -C xdk/lib/python3/dist-packages/osgeo -T -
needs:
- job: ubuntu-24-prepare
artifacts: true
## RedHat superbuild
redhat-prepare: extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-redhat-superbuild-base:8
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=redhat-8-gcc -DOTB_FULL_BUILD:BOOL=ON
redhat-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-redhat-superbuild-base:8
script:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=redhat-8-gcc
- ctest -V -S CI/superbuild_packaging.cmake -DIMAGE_NAME:string=redhat-8-gcc -DPACKAGE_BY_MODULE:BOOL=ON
- ctest -V -S CI/superbuild_packaging.cmake -DIMAGE_NAME:string=redhat-8-gcc -DPACKAGE_BY_MODULE:BOOL=OFF
# change package names to RedHat
- |
# use "find" to select only packages that matters
rh_packages=$(find build_packages -type f -name "OTB*Linux*.tar.gz")
for pkg in $rh_packages
do
new_name=$(sed "s/-Linux/-Linux_RedHat/g" <<< "$pkg")
mv $pkg $new_name
done
- mv build_packages/OTB-*.tar.gz . || true
needs:
- job: redhat-prepare
artifacts: true
## Windows
.windows-prepare:
extends: .common-prepare
before_script:
# This override the previous before_script
- set GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- set GIT_LFS_SKIP_SMUDGE=0
.windows-build:
extends: .common-build
before_script:
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
# Obviously Windows does not use same executable name as linux...
after_script:
# need to setup path to be able to use python
- .\CI\setup_python.ps1 x64
- python -u CI/cdash_handler.py
# - Win10
windows-10-prepare:
extends: .windows-prepare
tags:
- windows10
script:
- .\CI\dev_env.ps1 x64 xdk 10
- buildcache.exe -s
- ctest -C Release -VV -S CI/prepare_superbuild.cmake -DOTB_FULL_BUILD:BOOL=ON -DPython_ROOT_DIR:STRING="C:/tools/Python310-x64"
- buildcache.exe -s
windows-10-build:
extends: .windows-build
tags:
- windows10
script:
- .\CI\dev_env.ps1 x64 otb 10
- buildcache.exe -s
- ctest -V -S CI/main_superbuild.cmake -DPython_ROOT_DIR:STRING="C:/tools/Python310-x64"
- buildcache.exe -s
- .\CI\copy_boost_dlls.ps1
- ctest -V -S CI/superbuild_packaging.cmake -DPACKAGE_BY_MODULE:BOOL=OFF
- move "build_packages\OTB-*.zip" . || dir build_packages
needs: - job: windows-10-prepare
artifacts: true
#------------------------- QA related jobs -------------------------------------
ubuntu-qa-code-coverage:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:22.04
script:
- ctest -V -S CI/main_qa.cmake
-DIMAGE_NAME:string=ubuntu-22.04-llvm-qa
-DQA:BOOL=ON
- ./CI/otb_coverage.sh
- saxon-xslt -o build/ctest_report.xml
build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
CI/ctest2junit.xsl
needs:
- job: ubuntu-22-prepare
artifacts: true
ubuntu-qa-static-analysis:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:22.04
script:
- find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
grep -vE '^Modules/ThirdParty/' |
sort -u > header_directories
- cppcheck -j 8 -q --xml --xml-version=2 --enable=all
--language=c++ --std=c++14 --platform=unix64
--includes-file=header_directories
-i Modules/ThirdParty Modules
2> build/cppcheck_report.xml
after_script: []
needs:
- job: ubuntu-22-prepare
artifacts: true
ubuntu-sonar-report:
extends: .common
variables:
GIT_DEPTH: ""
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:22.04
stage: report
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
script:
- sonar-scanner -Dproject.settings=sonar-project.properties
-Dsonar.host.url=https://sonar.orfeo-toolbox.org
-Dsonar.login=$SONAR_OTB_TOKEN
-Dsonar.projectKey=$CI_PROJECT_NAMESPACE-$CI_PROJECT_NAME
`test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
after_script: []
needs:
- job: ubuntu-qa-code-coverage
artifacts: true
- job: ubuntu-qa-static-analysis
artifacts: true
#---------------------------- Deploy job ---------------------------------------
deploy:
tags:
- deploy
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
- develop@orfeotoolbox/otb
- /^release-[0-9]+\.[0-9]+$/
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
script:
- ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
needs:
- job: ubuntu-22-build
artifacts: true
- job: windows-10-build
artifacts: true
- job: redhat-build
artifacts: true
update-archive:
tags:
- deploy
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:22.04
stage: deploy
extends: .general
only:
- develop
- /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
script:
- ./CI/SuperbuildDownloadList.sh download build_archive
- ./CI/deploy-archive.sh build_archive
needs:
# don't push the archive if Superbuild was not successful
- job: ubuntu-22-prepare
artifacts: false
release-container:
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
refs:
- tags@orfeotoolbox/otb
variables:
- $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
script:
- curl --request POST
--form token=$K8S_SECRET_RELEASE
--form ref=master
--form variables[OTB_TAG]=$CI_COMMIT_TAG
https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline
release-docker:
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
stage: docker
only:
refs:
- /^release-[0-9]+\.[0-9]+$/
needs:
- job: ubuntu-22-build
artifacts: true
script:
# to authenticate in docker hub, kaniko needs a config file located at
# /kaniko/.docker/config.json
- mkdir -p /kaniko/.docker || true
- echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$(printf "%s:%s" "${DOCKER_HUB_USER}" "${DOCKER_HUB_PWD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
- /kaniko/executor --cleanup
--verbosity warn
--context $CI_PROJECT_DIR
--dockerfile $CI_PROJECT_DIR/Docker/Dockerfile_CI
--destination orfeotoolbox/otb:$CI_COMMIT_TAG