.gitlab-ci.yml 9.37 KB
Newer Older
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
1 2 3 4 5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
9 10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

Sébastien Dinot's avatar
Sébastien Dinot committed
11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
Sébastien Dinot's avatar
Sébastien Dinot committed
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

Sébastien Dinot's avatar
Sébastien Dinot committed
21
stages:
Guillaume Pasero's avatar
Guillaume Pasero committed
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
Sébastien Dinot's avatar
Sébastien Dinot committed
24
  - build
25
  - report
26
  - deploy
Sébastien Dinot's avatar
Sébastien Dinot committed
27

28
.general:
Sébastien Dinot's avatar
Sébastien Dinot committed
29 30 31 32 33 34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37 38 39 40 41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46 47 48 49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
Guillaume Pasero's avatar
Guillaume Pasero committed
50
    - ci_coverage
51
  stage: build
52 53 54 55 56 57 58 59 60
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - build/CookBook-*-html.tar.gz
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
61 62
      - build_packages/OTB-*.run
      - build_packages/OTB-*.zip
63 64 65
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
66

67
.common-prepare:
aregimbe's avatar
aregimbe committed
68
  extends: .general
69 70 71 72
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
Guillaume Pasero's avatar
Guillaume Pasero committed
73
    - ci_coverage
aregimbe's avatar
aregimbe committed
74 75
  stage: prepare
  before_script:
76 77 78
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
79 80
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
81
    when: always
aregimbe's avatar
aregimbe committed
82
    paths:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
83 84
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
85

86
#-------------------------- precheck job ---------------------------------------
Guillaume Pasero's avatar
Guillaume Pasero committed
87 88 89 90 91 92 93 94 95 96 97 98
#~ fast-build:
  #~ extends: .common
  #~ only: [merge_requests, branches]
  #~ stage: precheck
  #~ image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  #~ before_script:
    #~ - export GIT_LFS_SKIP_SMUDGE=1
    #~ - git checkout -f -q $CI_COMMIT_SHA
    #~ - python3 CI/check_twin_pipelines.py
  #~ script:
    #~ - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    #~ - ccache -s
99 100

#------------------------- prepare & build jobs --------------------------------
Guillaume Pasero's avatar
Guillaume Pasero committed
101 102 103 104 105
#~ debian-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
106

107
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
108
ubuntu-xdk-prepare:
109 110 111
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
112
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
113

Guillaume Pasero's avatar
Guillaume Pasero committed
114 115 116 117 118 119 120 121
#~ ubuntu-xdk-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
  #~ dependencies:
    #~ - ubuntu-xdk-prepare
122

Antoine Regimbeau's avatar
Antoine Regimbeau committed
123
## CentOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137
#~ centos-xdk-prepare:
  #~ extends: .common-prepare
  #~ image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  #~ script:
    #~ - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base

#~ centos-xdk-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
  #~ dependencies:
    #~ - centos-xdk-prepare
Cédric Traizet's avatar
Cédric Traizet committed
138

Antoine Regimbeau's avatar
Antoine Regimbeau committed
139
## MacOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
#~ macos-xdk-prepare:
  #~ extends: .common-prepare
  #~ tags:
    #~ - macos
  #~ script:
    #~ - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild

#~ macos-xdk-build:
  #~ tags:
    #~ - macos
  #~ extends: .common-build
  #~ script:
    #~ - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
    #~ - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
  #~ dependencies:
    #~ - macos-xdk-prepare
156

Guillaume Pasero's avatar
Guillaume Pasero committed
157
## Windows
Guillaume Pasero's avatar
Guillaume Pasero committed
158 159 160 161 162 163 164 165 166 167 168 169 170
#~ .windows-prepare:
  #~ extends: .common-prepare
  #~ before_script:
#~ # This override the previous before_script
    #~ - set GIT_LFS_SKIP_SMUDGE=1
    #~ - git checkout -f -q %CI_COMMIT_SHA%
    #~ - set GIT_LFS_SKIP_SMUDGE=0
#~ 
#~ .windows-build:
  #~ extends: .common-build
  #~ before_script:
    #~ - git lfs fetch origin %CI_COMMIT_SHA%
    #~ - git checkout -f -q %CI_COMMIT_SHA%
171

Guillaume Pasero's avatar
Guillaume Pasero committed
172
# - Win10
Guillaume Pasero's avatar
Guillaume Pasero committed
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
#~ windows-10-prepare:
  #~ extends: .windows-prepare
  #~ tags:
    #~ - windows10
  #~ script:
    #~ - call ./CI/dev_env.bat x64 xdk 10
    #~ - clcache.exe -s
    #~ - ctest -C Release -V -S CI/prepare_superbuild.cmake
    #~ - clcache.exe -s
#~ 
#~ windows-10-build:
  #~ extends: .windows-build
  #~ tags:
    #~ - windows10
  #~ script:
    #~ - call ./CI/dev_env.bat x64 otb 10
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_superbuild.cmake
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_packages.cmake
  #~ dependencies:
    #~ - windows-10-prepare
Guillaume Pasero's avatar
Guillaume Pasero committed
195

Guillaume Pasero's avatar
Guillaume Pasero committed
196
# - Win8.1
Guillaume Pasero's avatar
Guillaume Pasero committed
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
#~ windows-8-prepare:
  #~ extends: .windows-prepare
  #~ tags:
    #~ - windows8
  #~ script:
    #~ - call ./CI/dev_env.bat x86 xdk 8.1
    #~ - clcache.exe -s
    #~ - ctest -C Release -V -S CI/prepare_superbuild.cmake
    #~ - clcache.exe -s
#~ 
#~ windows-8-build:
  #~ extends: .windows-build
  #~ tags:
    #~ - windows8
  #~ script:
    #~ - call ./CI/dev_env.bat x86 otb 8.1
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_superbuild.cmake
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_packages.cmake
  #~ dependencies:
    #~ - windows-8-prepare
219

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
                     -DQA:BOOL=ON
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

274
#---------------------------- Deploy job ---------------------------------------
275 276 277 278 279 280 281
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
Guillaume Pasero's avatar
Guillaume Pasero committed
282 283
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
284 285 286 287 288
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
289
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
290
  dependencies:
291
    - ubuntu-xdk-build
292 293 294
    - centos-xdk-build
    - macos-xdk-build
    - windows-8-build
295
    - windows-10-build
296 297 298 299 300 301 302 303 304 305 306

release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
307 308 309 310 311
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline