.gitlab-ci.yml 12.1 KB
Newer Older
1
#
Sébastien Dinot's avatar
Sébastien Dinot committed
2
# Copyright (C) 2005-2020 Centre National d'Etudes Spatiales (CNES)
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#
# This file is part of Orfeo Toolbox
#
#     https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
21 22 23 24 25
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
26
# We have to use directly git-lfs wrapping commands to have an
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
27
# efficient cloning step.
28
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
29 30
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

Sébastien Dinot's avatar
Sébastien Dinot committed
31
variables:
32
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
33
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
Sébastien Dinot's avatar
Sébastien Dinot committed
34
  DOCKER_DRIVER: overlay2
35
  GIT_DEPTH: "3"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
36
  # Disable automatic checkout to let us fetch LFS before
37
  GIT_CHECKOUT: "false"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
38
  # The fetch strategy fails with LFS and GitLab
39
  GIT_STRATEGY: "clone"
40

Sébastien Dinot's avatar
Sébastien Dinot committed
41
stages:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
  - precheck
aregimbe's avatar
aregimbe committed
43
  - prepare
Sébastien Dinot's avatar
Sébastien Dinot committed
44
  - build
45
  - report
46
  - deploy
Sébastien Dinot's avatar
Sébastien Dinot committed
47

48
.general:
Sébastien Dinot's avatar
Sébastien Dinot committed
49 50 51 52 53 54
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
55
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
56
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
57 58 59 60 61
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
62
    - python3 -u CI/cdash_handler.py
63

Guillaume Pasero's avatar
Guillaume Pasero committed
64
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
65
  extends: .common
66 67 68 69
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
70
  stage: build
71 72 73 74 75 76
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
77 78 79 80 81
      - CookBook-*-html.tar.gz
      - CookBook-*.pdf
      - OTB-Doxygen-*.tar.bz2
      - OTB-*.run
      - OTB-*.zip
82 83 84
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
85
      - build/coverage_report.xml
86

87
.common-prepare:
aregimbe's avatar
aregimbe committed
88
  extends: .general
89 90 91 92
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
aregimbe's avatar
aregimbe committed
93 94
  stage: prepare
  before_script:
95 96 97
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
98 99
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
100
    when: always
aregimbe's avatar
aregimbe committed
101
    paths:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
102 103
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
Antoine Regimbeau's avatar
Antoine Regimbeau committed
104
      - build/*/*/*/*.cmake
aregimbe's avatar
aregimbe committed
105

106
#-------------------------- precheck job ---------------------------------------
107 108
fast-build:
  extends: .common
Sébastien Dinot's avatar
Sébastien Dinot committed
109 110 111
  only:
    - merge_requests
    - branches
112 113 114 115 116 117 118 119 120
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s
121

Sébastien Dinot's avatar
Sébastien Dinot committed
122
legal-check:
Sébastien Dinot's avatar
Sébastien Dinot committed
123
  extends: .common
124 125 126 127
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
Sébastien Dinot's avatar
Sébastien Dinot committed
128 129 130 131 132 133 134
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  variables:
    GIT_DEPTH: ""
  allow_failure: true
  script:
    - ./CI/contributors_check.sh
Sébastien Dinot's avatar
Sébastien Dinot committed
135
    - ./CI/headers_check.py
Sébastien Dinot's avatar
Sébastien Dinot committed
136 137
  after_script: []

138
#------------------------- prepare & build jobs --------------------------------
139 140
debian-build:
  extends: .common-build
141
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:testing
142
  allow_failure: true
143
  script:
144
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-testing-gcc
145 146 147
  after_script:
    - export CI_ALLOW_FAILURE=ON
    - python3 -u CI/cdash_handler.py
148 149
  needs:
    - job: fast-build
aregimbe's avatar
aregimbe committed
150

151
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
152
ubuntu-xdk-prepare:
153 154 155
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
156
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm
157

158
ubuntu-xdk-build-doc:
159 160 161
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
162
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
163 164 165
    - mv build/CookBook-*-html.tar.gz . || true
    - mv build/Documentation/Cookbook/latex/CookBook-*.pdf . || true
    - mv build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2 . || true
166
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk -DNAME_SUFFIX:string=-glibc-2.27
167
    - mv build_packages/OTB-*.run . || true
168 169 170
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true
171

Antoine Regimbeau's avatar
Antoine Regimbeau committed
172
## CentOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
173
centos-xdk-prepare:
174 175 176
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
177
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=centos-6.6-gcc
178

Guillaume Pasero's avatar
Guillaume Pasero committed
179
centos-xdk-build:
180 181 182
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
183 184
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=centos-6.6-gcc
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=centos-6.6-gcc
185
    - mv build_packages/OTB-*.run . || true
186 187 188
  needs:
    - job: centos-xdk-prepare
      artifacts: true
Cédric Traizet's avatar
Cédric Traizet committed
189

Antoine Regimbeau's avatar
Antoine Regimbeau committed
190
## MacOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
191
macos-xdk-prepare:
192
  extends: .common-prepare
193 194 195
  tags:
    - macos
  script:
196
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=macos-10.11.6-clang
197

Guillaume Pasero's avatar
Guillaume Pasero committed
198
macos-xdk-build:
199 200 201 202
  tags:
    - macos
  extends: .common-build
  script:
203 204
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=macos-10.11.6-clang
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=macos-10.11.6-clang
205
    - mv build_packages/OTB-*.run . || true
206 207 208
  needs:
    - job: macos-xdk-prepare
      artifacts: true
209

Guillaume Pasero's avatar
Guillaume Pasero committed
210
## Windows
211
.windows-prepare:
Guillaume Pasero's avatar
Guillaume Pasero committed
212
  extends: .common-prepare
aregimbe's avatar
aregimbe committed
213 214
  before_script:
# This override the previous before_script
215
    - set GIT_LFS_SKIP_SMUDGE=1
Guillaume Pasero's avatar
Guillaume Pasero committed
216
    - git checkout -f -q %CI_COMMIT_SHA%
217
    - set GIT_LFS_SKIP_SMUDGE=0
218

219
.windows-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
220 221 222
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
223
    - git checkout -f -q %CI_COMMIT_SHA%
224

Guillaume Pasero's avatar
Guillaume Pasero committed
225
# - Win10
226 227
windows-10-prepare:
  extends: .windows-prepare
228 229
  tags:
    - windows10
230
  script:
Guillaume Pasero's avatar
Guillaume Pasero committed
231
    - call ./CI/dev_env.bat x64 xdk 10
232 233 234 235
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

236
windows-10-build:
237
  extends: .windows-build
Guillaume Pasero's avatar
Guillaume Pasero committed
238
  tags:
239
    - windows10
Guillaume Pasero's avatar
Guillaume Pasero committed
240 241 242 243 244
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
Guillaume Pasero's avatar
Guillaume Pasero committed
245
    - ctest -V -S CI/main_packages.cmake
246
    - move "build_packages\OTB-*.zip" . || dir build_packages
247 248 249
  needs:
    - job: windows-10-prepare
      artifacts: true
Guillaume Pasero's avatar
Guillaume Pasero committed
250

Guillaume Pasero's avatar
Guillaume Pasero committed
251 252 253 254 255 256 257 258 259 260 261
# - Win8.1
windows-8-prepare:
  extends: .windows-prepare
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 xdk 8.1
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

Guillaume Pasero's avatar
Guillaume Pasero committed
262
windows-8-build:
263
  extends: .windows-build
Guillaume Pasero's avatar
Guillaume Pasero committed
264
  tags:
265
    - windows8
Guillaume Pasero's avatar
Guillaume Pasero committed
266
  script:
Guillaume Pasero's avatar
Guillaume Pasero committed
267
    - call ./CI/dev_env.bat x86 otb 8.1
Guillaume Pasero's avatar
Guillaume Pasero committed
268 269 270
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
Guillaume Pasero's avatar
Guillaume Pasero committed
271
    - ctest -V -S CI/main_packages.cmake
272
    - move "build_packages\OTB-*.zip" . || dir build_packages
273 274 275
  needs:
    - job: windows-8-prepare
      artifacts: true
276

277 278 279
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
280 281 282 283 284 285 286
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
287 288 289 290
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
291
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
292
                     -DQA:BOOL=ON
293
    - ./CI/otb_coverage.sh
294 295 296
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
297 298 299
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true
300

301 302
ubuntu-xdk-qa-static-analysis:
  extends: .common-build
303 304 305 306 307 308 309
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
310 311 312 313 314 315 316 317 318 319 320
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
321 322 323
  needs:
    - job: ubuntu-xdk-prepare
      artifacts: true
324 325 326 327 328 329 330 331

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
332 333 334 335 336 337
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
338 339 340 341
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
342
                    -Dsonar.projectKey=$CI_PROJECT_NAMESPACE-$CI_PROJECT_NAME
343 344 345 346
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
347 348 349 350 351
  needs:
    - job: ubuntu-xdk-qa-code-coverage
      artifacts: true
    - job: ubuntu-xdk-qa-static-analysis
      artifacts: true
352

353 354 355 356 357
#------------------------- Conda packages jobs ---------------------------------
conda-linux-build:
  extends: .common
  stage: build
  only:
358 359
    - nightly
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
360 361 362 363 364
  image: $BUILD_IMAGE_REGISTRY/otb-conda-build:latest
  allow_failure: true
  script:
    - export otb_tag=${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA}
    - ./CI/conda_build.sh
365
  after_script: []
366 367
  needs:
    - job: fast-build
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
      artifacts: false
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - conda-bld/*

conda-linux-deploy:
  extends: .general
  stage: deploy
  only:
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
  image: $BUILD_IMAGE_REGISTRY/otb-conda-build:latest
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
    - ./CI/conda_deploy.sh
  needs:
    - job: conda-linux-build
      artifacts: yes
390

391
#---------------------------- Deploy job ---------------------------------------
392 393 394 395 396 397 398
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
399 400
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
401 402 403 404 405
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
406 407 408 409 410 411 412 413 414 415 416 417
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
  needs:
    - job: ubuntu-xdk-build-doc
      artifacts: true
    - job: centos-xdk-build
      artifacts: true
    - job: macos-xdk-build
      artifacts: true
    - job: windows-8-build
      artifacts: true
    - job: windows-10-build
      artifacts: true
guillaume pernot's avatar
guillaume pernot committed
418

419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline