.gitlab-ci.yml 9.39 KB
Newer Older
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
1 2 3 4 5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
9 10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

Sébastien Dinot's avatar
Sébastien Dinot committed
11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
Sébastien Dinot's avatar
Sébastien Dinot committed
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

Sébastien Dinot's avatar
Sébastien Dinot committed
21
stages:
Guillaume Pasero's avatar
Guillaume Pasero committed
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
Sébastien Dinot's avatar
Sébastien Dinot committed
24
  - build
25
  - report
26
  - deploy
Sébastien Dinot's avatar
Sébastien Dinot committed
27

28
.general:
Sébastien Dinot's avatar
Sébastien Dinot committed
29 30 31 32 33 34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37 38 39 40 41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46 47 48 49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
50
  stage: build
51 52 53 54 55 56 57 58 59
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - build/CookBook-*-html.tar.gz
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
60 61
      - build_packages/OTB-*.run
      - build_packages/OTB-*.zip
62 63 64
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
65
      - build/coverage_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
66

67
.common-prepare:
aregimbe's avatar
aregimbe committed
68
  extends: .general
69 70 71 72
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
aregimbe's avatar
aregimbe committed
73 74
  stage: prepare
  before_script:
75 76 77
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
78 79
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
80
    when: always
aregimbe's avatar
aregimbe committed
81
    paths:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
82 83
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
84

85
#-------------------------- precheck job ---------------------------------------
86 87 88 89 90 91 92 93 94 95 96 97
fast-build:
  extends: .common
  only: [merge_requests, branches]
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s
98 99

#------------------------- prepare & build jobs --------------------------------
100 101 102 103 104 105
debian-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
  dependencies: []
106

107
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
108
ubuntu-xdk-prepare:
109 110 111
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
112
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
113

114 115 116 117 118 119 120 121
ubuntu-xdk-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
  dependencies:
    - ubuntu-xdk-prepare
122

Antoine Regimbeau's avatar
Antoine Regimbeau committed
123
## CentOS superbuild
124 125 126 127 128
centos-xdk-prepare:
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
Guillaume Pasero's avatar
Guillaume Pasero committed
129

130 131 132 133 134 135 136 137
centos-xdk-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
  dependencies:
    - centos-xdk-prepare
Cédric Traizet's avatar
Cédric Traizet committed
138

Antoine Regimbeau's avatar
Antoine Regimbeau committed
139
## MacOS superbuild
140 141 142 143 144 145
macos-xdk-prepare:
  extends: .common-prepare
  tags:
    - macos
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
146

147 148 149 150 151 152 153 154 155
macos-xdk-build:
  tags:
    - macos
  extends: .common-build
  script:
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
  dependencies:
    - macos-xdk-prepare
156

Guillaume Pasero's avatar
Guillaume Pasero committed
157
## Windows
158 159 160 161 162 163 164 165 166 167 168 169 170
.windows-prepare:
  extends: .common-prepare
  before_script:
# This override the previous before_script
    - set GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q %CI_COMMIT_SHA%
    - set GIT_LFS_SKIP_SMUDGE=0

.windows-build:
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
    - git checkout -f -q %CI_COMMIT_SHA%
171

Guillaume Pasero's avatar
Guillaume Pasero committed
172
# - Win10
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
windows-10-prepare:
  extends: .windows-prepare
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 xdk 10
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

windows-10-build:
  extends: .windows-build
  tags:
    - windows10
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
    - ctest -V -S CI/main_packages.cmake
  dependencies:
    - windows-10-prepare
Guillaume Pasero's avatar
Guillaume Pasero committed
195

Guillaume Pasero's avatar
Guillaume Pasero committed
196
# - Win8.1
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
windows-8-prepare:
  extends: .windows-prepare
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 xdk 8.1
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

windows-8-build:
  extends: .windows-build
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 otb 8.1
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
    - ctest -V -S CI/main_packages.cmake
  dependencies:
    - windows-8-prepare
219

220 221 222
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
223 224 225 226 227 228 229
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
230 231 232 233
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
234
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
235
                     -DQA:BOOL=ON
236
    - ./CI/otb_coverage.sh
237 238 239 240 241 242 243 244
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
245 246 247 248 249 250 251
  only:
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
273 274 275 276 277 278
    refs:
      - merge_requests
      - develop
      - /^release-[0-9]+\.[0-9]+$/
    variables:
      - $SONAR_OTB_TOKEN
279 280 281 282 283 284 285 286 287 288 289 290
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

291
#---------------------------- Deploy job ---------------------------------------
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
  dependencies:
    - ubuntu-xdk-build
    - centos-xdk-build
    - macos-xdk-build
    - windows-8-build
    - windows-10-build

release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline