.gitlab-ci.yml 9.02 KB
Newer Older
1 2 3 4 5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
9 10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

21
stages:
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
24
  - build
25
  - report
26
  - deploy
27

28
.general:
29 30 31 32 33 34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37 38 39 40 41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46 47 48 49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
50
  stage: build
51 52 53 54 55 56 57 58 59
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - build/CookBook-*-html.tar.gz
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
60 61
      - build_packages/OTB-*.run
      - build_packages/OTB-*.zip
62 63 64
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
65

66
.common-prepare:
aregimbe's avatar
aregimbe committed
67
  extends: .general
68 69 70 71
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
aregimbe's avatar
aregimbe committed
72 73
  stage: prepare
  before_script:
74 75 76
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
77 78
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
79
    when: always
aregimbe's avatar
aregimbe committed
80
    paths:
81 82
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
83

84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
#-------------------------- precheck job ---------------------------------------
fast-build:
  extends: .common
  only: [merge_requests, branches]
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s

#------------------------- prepare & build jobs --------------------------------
debian-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc

105
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
106
ubuntu-xdk-prepare:
107 108 109
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
110
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
111

Guillaume Pasero's avatar
Guillaume Pasero committed
112
ubuntu-xdk-build:
113 114 115
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
116
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
117
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
118
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
119
    - ubuntu-xdk-prepare
120

Antoine Regimbeau's avatar
Antoine Regimbeau committed
121
## CentOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
122
centos-xdk-prepare:
123 124 125 126 127
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base

Guillaume Pasero's avatar
Guillaume Pasero committed
128
centos-xdk-build:
129 130 131 132
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
133
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
134
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
135
    - centos-xdk-prepare
136

Antoine Regimbeau's avatar
Antoine Regimbeau committed
137
## MacOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
138
macos-xdk-prepare:
139
  extends: .common-prepare
140 141 142 143 144
  tags:
    - macos
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild

Guillaume Pasero's avatar
Guillaume Pasero committed
145
macos-xdk-build:
146 147 148 149 150
  tags:
    - macos
  extends: .common-build
  script:
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
151
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
152
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
153
    - macos-xdk-prepare
154

155
## Windows
156
.windows-prepare:
Guillaume Pasero's avatar
Guillaume Pasero committed
157
  extends: .common-prepare
aregimbe's avatar
aregimbe committed
158 159
  before_script:
# This override the previous before_script
160
    - set GIT_LFS_SKIP_SMUDGE=1
Guillaume Pasero's avatar
Guillaume Pasero committed
161
    - git checkout -f -q %CI_COMMIT_SHA%
162
    - set GIT_LFS_SKIP_SMUDGE=0
163

164
.windows-build:
165 166 167
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
168
    - git checkout -f -q %CI_COMMIT_SHA%
169

170
# - Win10
171 172
windows-10-prepare:
  extends: .windows-prepare
173 174
  tags:
    - windows10
175
  script:
Guillaume Pasero's avatar
Guillaume Pasero committed
176
    - call ./CI/dev_env.bat x64 xdk 10
177 178 179 180
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

181
windows-10-build:
182
  extends: .windows-build
183
  tags:
184
    - windows10
185 186 187 188 189
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
190
    - ctest -V -S CI/main_packages.cmake
191 192 193
  dependencies:
    - windows-10-prepare

194 195 196 197 198 199 200 201 202 203 204
# - Win8.1
windows-8-prepare:
  extends: .windows-prepare
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 xdk 8.1
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

205
windows-8-build:
206
  extends: .windows-build
207
  tags:
208
    - windows8
209
  script:
210
    - call ./CI/dev_env.bat x86 otb 8.1
211 212 213
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
214
    - ctest -V -S CI/main_packages.cmake
215 216
  dependencies:
    - windows-8-prepare
217

218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
                     -DQA:BOOL=ON
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

272
#---------------------------- Deploy job ---------------------------------------
273 274 275 276 277 278 279
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
280 281
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
282 283 284 285 286
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
287
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
288
  dependencies:
289
    - ubuntu-xdk-build
290 291 292
    - centos-xdk-build
    - macos-xdk-build
    - windows-8-build
293
    - windows-10-build
294 295 296 297 298 299 300 301 302 303 304

release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
305 306 307 308 309
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline