.gitlab-ci.yml 9.73 KB
Newer Older
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
1 2 3 4 5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
9 10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

Sébastien Dinot's avatar
Sébastien Dinot committed
11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
Sébastien Dinot's avatar
Sébastien Dinot committed
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

Sébastien Dinot's avatar
Sébastien Dinot committed
21
stages:
Guillaume Pasero's avatar
Guillaume Pasero committed
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
Sébastien Dinot's avatar
Sébastien Dinot committed
24
  - build
25
  - report
26
  - deploy
Sébastien Dinot's avatar
Sébastien Dinot committed
27

28
.general:
Sébastien Dinot's avatar
Sébastien Dinot committed
29 30 31 32 33 34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37 38 39 40 41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46 47 48 49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
50
  stage: build
51 52 53 54 55 56
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
57 58 59 60 61
      - CookBook-*-html.tar.gz
      - CookBook-*.pdf
      - OTB-Doxygen-*.tar.bz2
      - OTB-*.run
      - OTB-*.zip
62 63 64
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
65
      - build/coverage_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
66

67
.common-prepare:
aregimbe's avatar
aregimbe committed
68
  extends: .general
69 70 71 72
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
aregimbe's avatar
aregimbe committed
73 74
  stage: prepare
  before_script:
75 76 77
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
78 79
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
80
    when: always
aregimbe's avatar
aregimbe committed
81
    paths:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
82 83
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
84

85
#-------------------------- precheck job ---------------------------------------
86 87 88 89 90 91 92 93 94 95 96 97
fast-build:
  extends: .common
  only: [merge_requests, branches]
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s
98 99

#------------------------- prepare & build jobs --------------------------------
100 101 102 103 104 105
debian-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
  dependencies: []
106

107
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
108
ubuntu-xdk-prepare:
109 110 111
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
112
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
113

114
ubuntu-xdk-build-doc:
115 116 117 118
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
119 120 121
    - mv build/CookBook-*-html.tar.gz . || true
    - mv build/Documentation/Cookbook/latex/CookBook-*.pdf . || true
    - mv build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 . || true
122
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
123
    - mv build_packages/OTB-*.run . || true
124 125
  dependencies:
    - ubuntu-xdk-prepare
126

Antoine Regimbeau's avatar
Antoine Regimbeau committed
127
## CentOS superbuild
128 129 130 131 132
centos-xdk-prepare:
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
Guillaume Pasero's avatar
Guillaume Pasero committed
133

134 135 136 137 138 139
centos-xdk-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
140
    - mv build_packages/OTB-*.run . || true
141 142
  dependencies:
    - centos-xdk-prepare
Cédric Traizet's avatar
Cédric Traizet committed
143

Antoine Regimbeau's avatar
Antoine Regimbeau committed
144
## MacOS superbuild
145 146 147 148 149 150
macos-xdk-prepare:
  extends: .common-prepare
  tags:
    - macos
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
151

152 153 154 155 156 157 158
macos-xdk-build:
  tags:
    - macos
  extends: .common-build
  script:
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
159
    - mv build_packages/OTB-*.run . || true
160 161
  dependencies:
    - macos-xdk-prepare
162

Guillaume Pasero's avatar
Guillaume Pasero committed
163
## Windows
164 165 166 167 168 169 170 171 172 173 174 175 176
.windows-prepare:
  extends: .common-prepare
  before_script:
# This override the previous before_script
    - set GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q %CI_COMMIT_SHA%
    - set GIT_LFS_SKIP_SMUDGE=0

.windows-build:
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
    - git checkout -f -q %CI_COMMIT_SHA%
177

Guillaume Pasero's avatar
Guillaume Pasero committed
178
# - Win10
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
windows-10-prepare:
  extends: .windows-prepare
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 xdk 10
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

windows-10-build:
  extends: .windows-build
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
    - ctest -V -S CI/main_packages.cmake
199
    - move "build_packages\OTB-*.zip" . || dir build_packages
200 201
  dependencies:
    - windows-10-prepare
Guillaume Pasero's avatar
Guillaume Pasero committed
202

Guillaume Pasero's avatar
Guillaume Pasero committed
203
# - Win8.1
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
windows-8-prepare:
  extends: .windows-prepare
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 xdk 8.1
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

windows-8-build:
  extends: .windows-build
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 otb 8.1
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
    - ctest -V -S CI/main_packages.cmake
224
    - move "build_packages\OTB-*.zip" . || dir build_packages
225 226
  dependencies:
    - windows-8-prepare
227

228 229 230
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
231 232 233 234 235 236 237
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
238 239 240 241
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
242
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
243
                     -DQA:BOOL=ON
244
    - ./CI/otb_coverage.sh
245 246 247 248 249 250 251 252
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
253 254 255 256 257 258 259
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
281 282 283 284 285 286
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
287 288 289 290 291 292 293 294 295 296 297 298
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

299
#---------------------------- Deploy job ---------------------------------------
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
  dependencies:
316
    - ubuntu-xdk-build-doc
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
    - centos-xdk-build
    - macos-xdk-build
    - windows-8-build
    - windows-10-build

release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline