.gitlab-ci.yml 9.62 KB
Newer Older
1 2 3 4 5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
9 10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

21
stages:
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
24
  - build
25
  - report
26
  - deploy
27

28
.general:
29 30 31 32 33 34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37 38 39 40 41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46 47 48 49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
50
    - ci_coverage
51
  stage: build
52 53 54 55 56 57 58 59 60
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - build/CookBook-*-html.tar.gz
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
61 62
      - build_packages/OTB-*.run
      - build_packages/OTB-*.zip
63 64 65
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
66
      - build/coverage_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
67

68
.common-prepare:
aregimbe's avatar
aregimbe committed
69
  extends: .general
70 71 72 73
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
74
    - ci_coverage
aregimbe's avatar
aregimbe committed
75 76
  stage: prepare
  before_script:
77 78 79
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
80 81
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
82
    when: always
aregimbe's avatar
aregimbe committed
83
    paths:
84 85
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
86

87
#-------------------------- precheck job ---------------------------------------
88 89 90 91 92 93 94 95 96 97 98 99
#~ fast-build:
  #~ extends: .common
  #~ only: [merge_requests, branches]
  #~ stage: precheck
  #~ image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  #~ before_script:
    #~ - export GIT_LFS_SKIP_SMUDGE=1
    #~ - git checkout -f -q $CI_COMMIT_SHA
    #~ - python3 CI/check_twin_pipelines.py
  #~ script:
    #~ - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    #~ - ccache -s
100 101

#------------------------- prepare & build jobs --------------------------------
102 103 104 105 106
#~ debian-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
107

108
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
109
ubuntu-xdk-prepare:
110 111 112
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
113
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
114

115 116 117 118 119 120 121 122
#~ ubuntu-xdk-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
  #~ dependencies:
    #~ - ubuntu-xdk-prepare
123

Antoine Regimbeau's avatar
Antoine Regimbeau committed
124
## CentOS superbuild
125 126 127 128 129 130 131 132 133 134 135 136 137 138
#~ centos-xdk-prepare:
  #~ extends: .common-prepare
  #~ image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  #~ script:
    #~ - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base

#~ centos-xdk-build:
  #~ extends: .common-build
  #~ image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  #~ script:
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
    #~ - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
  #~ dependencies:
    #~ - centos-xdk-prepare
139

Antoine Regimbeau's avatar
Antoine Regimbeau committed
140
## MacOS superbuild
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
#~ macos-xdk-prepare:
  #~ extends: .common-prepare
  #~ tags:
    #~ - macos
  #~ script:
    #~ - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild

#~ macos-xdk-build:
  #~ tags:
    #~ - macos
  #~ extends: .common-build
  #~ script:
    #~ - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
    #~ - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
  #~ dependencies:
    #~ - macos-xdk-prepare
157

158
## Windows
159 160 161 162 163 164 165 166 167 168 169 170 171
#~ .windows-prepare:
  #~ extends: .common-prepare
  #~ before_script:
#~ # This override the previous before_script
    #~ - set GIT_LFS_SKIP_SMUDGE=1
    #~ - git checkout -f -q %CI_COMMIT_SHA%
    #~ - set GIT_LFS_SKIP_SMUDGE=0
#~ 
#~ .windows-build:
  #~ extends: .common-build
  #~ before_script:
    #~ - git lfs fetch origin %CI_COMMIT_SHA%
    #~ - git checkout -f -q %CI_COMMIT_SHA%
172

173
# - Win10
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
#~ windows-10-prepare:
  #~ extends: .windows-prepare
  #~ tags:
    #~ - windows10
  #~ script:
    #~ - call ./CI/dev_env.bat x64 xdk 10
    #~ - clcache.exe -s
    #~ - ctest -C Release -V -S CI/prepare_superbuild.cmake
    #~ - clcache.exe -s
#~ 
#~ windows-10-build:
  #~ extends: .windows-build
  #~ tags:
    #~ - windows10
  #~ script:
    #~ - call ./CI/dev_env.bat x64 otb 10
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_superbuild.cmake
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_packages.cmake
  #~ dependencies:
    #~ - windows-10-prepare
196

197
# - Win8.1
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
#~ windows-8-prepare:
  #~ extends: .windows-prepare
  #~ tags:
    #~ - windows8
  #~ script:
    #~ - call ./CI/dev_env.bat x86 xdk 8.1
    #~ - clcache.exe -s
    #~ - ctest -C Release -V -S CI/prepare_superbuild.cmake
    #~ - clcache.exe -s
#~ 
#~ windows-8-build:
  #~ extends: .windows-build
  #~ tags:
    #~ - windows8
  #~ script:
    #~ - call ./CI/dev_env.bat x86 otb 8.1
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_superbuild.cmake
    #~ - clcache.exe -s
    #~ - ctest -V -S CI/main_packages.cmake
  #~ dependencies:
    #~ - windows-8-prepare
220

221 222 223 224 225 226 227
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
228
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
229
                     -DQA:BOOL=ON
Guillaume Pasero's avatar
Guillaume Pasero committed
230
    - ls -lh build/cov_data
Guillaume Pasero's avatar
Guillaume Pasero committed
231
    - gcovr -x -o build/coverage_report.xml build/cov_data
232 233 234
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
Guillaume Pasero's avatar
Guillaume Pasero committed
235
  #~ after_script: []
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
265
    - ci_coverage
266 267 268 269 270 271 272 273 274 275 276 277
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

278
#---------------------------- Deploy job ---------------------------------------
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
#~ deploy:
  #~ tags:
    #~ - deploy
  #~ image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  #~ stage: deploy
  #~ extends: .general
  #~ only:
    #~ - develop@orfeotoolbox/otb
    #~ - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
  #~ before_script:
    #~ # Provision efficiently the local LFS cache before checkout
    #~ - git lfs fetch origin $CI_COMMIT_SHA
    #~ - git checkout -f -q $CI_COMMIT_SHA
  #~ script:
    #~ - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
  #~ dependencies:
    #~ - ubuntu-xdk-build
    #~ - centos-xdk-build
    #~ - macos-xdk-build
    #~ - windows-8-build
    #~ - windows-10-build
#~ 
#~ release-container:
  #~ image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  #~ stage: deploy
  #~ extends: .general
  #~ only:
    #~ refs:
      #~ - tags@orfeotoolbox/otb
    #~ variables:
      #~ - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  #~ script:
    #~ - curl --request POST
           #~ --form token=$K8S_SECRET_RELEASE
           #~ --form ref=master
           #~ --form variables[OTB_TAG]=$CI_COMMIT_TAG
           #~ https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline