.gitlab-ci.yml 9.02 KB
Newer Older
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
1
2
3
4
5
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
6
# We have to use directly git-lfs wrapping commands to have an
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
7
# efficient cloning step.
8
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
9
10
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

Sébastien Dinot's avatar
Sébastien Dinot committed
11
variables:
12
  BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
13
  GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
Sébastien Dinot's avatar
Sébastien Dinot committed
14
  DOCKER_DRIVER: overlay2
15
  GIT_DEPTH: "3"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
16
  # Disable automatic checkout to let us fetch LFS before
17
  GIT_CHECKOUT: "false"
Guilhem Bonnefille's avatar
Guilhem Bonnefille committed
18
  # The fetch strategy fails with LFS and GitLab
19
  GIT_STRATEGY: "clone"
20

Sébastien Dinot's avatar
Sébastien Dinot committed
21
stages:
Guillaume Pasero's avatar
Guillaume Pasero committed
22
  - precheck
aregimbe's avatar
aregimbe committed
23
  - prepare
Sébastien Dinot's avatar
Sébastien Dinot committed
24
  - build
25
  - report
26
  - deploy
Sébastien Dinot's avatar
Sébastien Dinot committed
27

28
.general:
Sébastien Dinot's avatar
Sébastien Dinot committed
29
30
31
32
33
34
  retry:
    max: 2
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

Guillaume Pasero's avatar
Guillaume Pasero committed
35
.common:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
36
  extends: .general
Guillaume Pasero's avatar
Guillaume Pasero committed
37
38
39
40
41
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  after_script:
Guillaume Pasero's avatar
Guillaume Pasero committed
42
    - python3 -u CI/cdash_handler.py
Guillaume Pasero's avatar
Guillaume Pasero committed
43

Guillaume Pasero's avatar
Guillaume Pasero committed
44
.common-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
45
  extends: .common
46
47
48
49
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
50
  stage: build
51
52
53
54
55
56
57
58
59
  artifacts:
    when: always
    expire_in: 24 hrs
    paths:
      - build/*/*.log #CMake log
      - log/*.txt # Others
      - build/CookBook-*-html.tar.gz
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
60
61
      - build_packages/OTB-*.run
      - build_packages/OTB-*.zip
62
63
64
      - build/compile_commands.json
      - build/ctest_report.xml
      - build/cppcheck_report.xml
Guillaume Pasero's avatar
Guillaume Pasero committed
65

66
.common-prepare:
aregimbe's avatar
aregimbe committed
67
  extends: .general
68
69
70
71
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
aregimbe's avatar
aregimbe committed
72
73
  stage: prepare
  before_script:
74
75
76
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - export GIT_LFS_SKIP_SMUDGE=0
aregimbe's avatar
aregimbe committed
77
78
  artifacts:
    expire_in: 24 hrs
Antoine Regimbeau's avatar
Antoine Regimbeau committed
79
    when: always
aregimbe's avatar
aregimbe committed
80
    paths:
Antoine Regimbeau's avatar
Antoine Regimbeau committed
81
82
      - sb_branch.txt # Needed to checkout correct branch in build step
      - build/*/*/*/*.log # Superbuild log
aregimbe's avatar
aregimbe committed
83

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#-------------------------- precheck job ---------------------------------------
fast-build:
  extends: .common
  only: [merge_requests, branches]
  stage: precheck
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
  before_script:
    - export GIT_LFS_SKIP_SMUDGE=1
    - git checkout -f -q $CI_COMMIT_SHA
    - python3 CI/check_twin_pipelines.py
  script:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s

#------------------------- prepare & build jobs --------------------------------
debian-build:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc

105
## Ubuntu superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
106
ubuntu-xdk-prepare:
107
108
109
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
110
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
111

Guillaume Pasero's avatar
Guillaume Pasero committed
112
ubuntu-xdk-build:
113
114
115
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
116
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
117
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
118
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
119
    - ubuntu-xdk-prepare
120

Antoine Regimbeau's avatar
Antoine Regimbeau committed
121
## CentOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
122
centos-xdk-prepare:
123
124
125
126
127
  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base

Guillaume Pasero's avatar
Guillaume Pasero committed
128
centos-xdk-build:
129
130
131
132
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
133
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
134
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
135
    - centos-xdk-prepare
Cédric Traizet's avatar
Cédric Traizet committed
136

Antoine Regimbeau's avatar
Antoine Regimbeau committed
137
## MacOS superbuild
Guillaume Pasero's avatar
Guillaume Pasero committed
138
macos-xdk-prepare:
139
  extends: .common-prepare
140
141
142
143
144
  tags:
    - macos
  script:
    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild

Guillaume Pasero's avatar
Guillaume Pasero committed
145
macos-xdk-build:
146
147
148
149
150
  tags:
    - macos
  extends: .common-build
  script:
    - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
151
    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
152
  dependencies:
Guillaume Pasero's avatar
Guillaume Pasero committed
153
    - macos-xdk-prepare
154

Guillaume Pasero's avatar
Guillaume Pasero committed
155
## Windows
156
.windows-prepare:
Guillaume Pasero's avatar
Guillaume Pasero committed
157
  extends: .common-prepare
aregimbe's avatar
aregimbe committed
158
159
  before_script:
# This override the previous before_script
160
    - set GIT_LFS_SKIP_SMUDGE=1
Guillaume Pasero's avatar
Guillaume Pasero committed
161
    - git checkout -f -q %CI_COMMIT_SHA%
162
    - set GIT_LFS_SKIP_SMUDGE=0
163

164
.windows-build:
Guillaume Pasero's avatar
Guillaume Pasero committed
165
166
167
  extends: .common-build
  before_script:
    - git lfs fetch origin %CI_COMMIT_SHA%
168
    - git checkout -f -q %CI_COMMIT_SHA%
169

Guillaume Pasero's avatar
Guillaume Pasero committed
170
# - Win10
171
172
windows-10-prepare:
  extends: .windows-prepare
173
174
  tags:
    - windows10
175
  script:
Guillaume Pasero's avatar
Guillaume Pasero committed
176
    - call ./CI/dev_env.bat x64 xdk 10
177
178
179
180
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

181
windows-10-build:
182
  extends: .windows-build
Guillaume Pasero's avatar
Guillaume Pasero committed
183
  tags:
184
    - windows10
Guillaume Pasero's avatar
Guillaume Pasero committed
185
186
187
188
189
  script:
    - call ./CI/dev_env.bat x64 otb 10
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
Guillaume Pasero's avatar
Guillaume Pasero committed
190
    - ctest -V -S CI/main_packages.cmake
Guillaume Pasero's avatar
Guillaume Pasero committed
191
192
193
  dependencies:
    - windows-10-prepare

Guillaume Pasero's avatar
Guillaume Pasero committed
194
195
196
197
198
199
200
201
202
203
204
# - Win8.1
windows-8-prepare:
  extends: .windows-prepare
  tags:
    - windows8
  script:
    - call ./CI/dev_env.bat x86 xdk 8.1
    - clcache.exe -s
    - ctest -C Release -V -S CI/prepare_superbuild.cmake
    - clcache.exe -s

Guillaume Pasero's avatar
Guillaume Pasero committed
205
windows-8-build:
206
  extends: .windows-build
Guillaume Pasero's avatar
Guillaume Pasero committed
207
  tags:
208
    - windows8
Guillaume Pasero's avatar
Guillaume Pasero committed
209
  script:
Guillaume Pasero's avatar
Guillaume Pasero committed
210
    - call ./CI/dev_env.bat x86 otb 8.1
Guillaume Pasero's avatar
Guillaume Pasero committed
211
212
213
    - clcache.exe -s
    - ctest -V -S CI/main_superbuild.cmake
    - clcache.exe -s
Guillaume Pasero's avatar
Guillaume Pasero committed
214
    - ctest -V -S CI/main_packages.cmake
Guillaume Pasero's avatar
Guillaume Pasero committed
215
216
  dependencies:
    - windows-8-prepare
217

218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
               ctest -V -S CI/main_qa.cmake
                     -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
                     -DQA:BOOL=ON
    - saxon-xslt -o build/ctest_report.xml
                 build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
                 CI/ctest2junit.xsl
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-qa-static-analysis:
  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  script:
    - find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
          grep -vE '^Modules/ThirdParty/' |
          sort -u > header_directories
    - cppcheck -j 8 -q --xml --xml-version=2 --enable=all
               --language=c++ --std=c++14 --platform=unix64
               --includes-file=header_directories
               -i Modules/ThirdParty Modules
               2> build/cppcheck_report.xml
  after_script: []
  dependencies:
    - ubuntu-xdk-prepare

ubuntu-xdk-report:
  extends: .common
  variables:
    GIT_DEPTH: ""
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
  stage: report
  only:
    - merge_requests
    - develop
    - /^release-[0-9]+\.[0-9]+$/
  script:
    - sonar-scanner -Dproject.settings=sonar-project.properties
                    -Dsonar.host.url=https://sonar.orfeo-toolbox.org
                    -Dsonar.login=$SONAR_OTB_TOKEN
                    `test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
                    -Dsonar.branch.name=$CI_COMMIT_REF_NAME

  after_script: []
  dependencies:
    - ubuntu-xdk-qa-code-coverage
    - ubuntu-xdk-qa-static-analysis

272
#---------------------------- Deploy job ---------------------------------------
273
274
275
276
277
278
279
deploy:
  tags:
    - deploy
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
Guillaume Pasero's avatar
Guillaume Pasero committed
280
281
    - develop@orfeotoolbox/otb
    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
282
283
284
285
286
  before_script:
    # Provision efficiently the local LFS cache before checkout
    - git lfs fetch origin $CI_COMMIT_SHA
    - git checkout -f -q $CI_COMMIT_SHA
  script:
287
    - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
288
  dependencies:
289
    - ubuntu-xdk-build
290
291
292
    - centos-xdk-build
    - macos-xdk-build
    - windows-8-build
293
    - windows-10-build
294
295
296
297
298
299
300
301
302
303
304

release-container:
  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
  stage: deploy
  extends: .general
  only:
    refs:
      - tags@orfeotoolbox/otb
    variables:
      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
  script:
305
306
307
308
309
    - curl --request POST
           --form token=$K8S_SECRET_RELEASE
           --form ref=master
           --form variables[OTB_TAG]=$CI_COMMIT_TAG
           https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline