Commit c6083ff4 authored by Guillaume Pasero's avatar Guillaume Pasero
Browse files

MRG: Synchronize with branch develop

Conflicts:
	Modules/Wrappers/ApplicationEngine/include/otbWrapperParameterList.h
	Modules/Wrappers/ApplicationEngine/src/otbWrapperApplication.cxx
parents cac0926d 2f15f2e6
Data/**/*.[0-9][0-9][0-9] filter=lfs diff=lfs merge=lfs -text
Data/**/*.BIL filter=lfs diff=lfs merge=lfs -text
Data/**/*.bsq filter=lfs diff=lfs merge=lfs -text
Data/**/*.c[0-9] filter=lfs diff=lfs merge=lfs -text
Data/**/*.c[0-9][0-9] filter=lfs diff=lfs merge=lfs -text
Data/**/*.ci4 filter=lfs diff=lfs merge=lfs -text
Data/**/*.cr4 filter=lfs diff=lfs merge=lfs -text
Data/**/*.dat filter=lfs diff=lfs merge=lfs -text
Data/**/*.dbf filter=lfs diff=lfs merge=lfs -text
Data/**/*.grd filter=lfs diff=lfs merge=lfs -text
Data/**/*.hdf filter=lfs diff=lfs merge=lfs -text
Data/**/*.he5 filter=lfs diff=lfs merge=lfs -text
Data/**/*.hgt filter=lfs diff=lfs merge=lfs -text
Data/**/*.id filter=lfs diff=lfs merge=lfs -text
Data/**/*.idx filter=lfs diff=lfs merge=lfs -text
Data/**/*.ima filter=lfs diff=lfs merge=lfs -text
Data/**/*.img filter=lfs diff=lfs merge=lfs -text
Data/**/*.index filter=lfs diff=lfs merge=lfs -text
Data/**/*.j2k filter=lfs diff=lfs merge=lfs -text
Data/**/*.jp2 filter=lfs diff=lfs merge=lfs -text
Data/**/*.jpeg filter=lfs diff=lfs merge=lfs -text
Data/**/*.jpg filter=lfs diff=lfs merge=lfs -text
Data/**/*.kmz filter=lfs diff=lfs merge=lfs -text
Data/**/*.las filter=lfs diff=lfs merge=lfs -text
Data/**/*.lum filter=lfs diff=lfs merge=lfs -text
Data/**/*.map filter=lfs diff=lfs merge=lfs -text
Data/**/*.mw filter=lfs diff=lfs merge=lfs -text
Data/**/*.pack filter=lfs diff=lfs merge=lfs -text
Data/**/*.pix filter=lfs diff=lfs merge=lfs -text
Data/**/*.png filter=lfs diff=lfs merge=lfs -text
Data/**/*.raw filter=lfs diff=lfs merge=lfs -text
Data/**/*.shp filter=lfs diff=lfs merge=lfs -text
Data/**/*.shx filter=lfs diff=lfs merge=lfs -text
Data/**/*.sqlite filter=lfs diff=lfs merge=lfs -text
# Data/**/*.tar filter=lfs diff=lfs merge=lfs -text
Data/**/*.tif filter=lfs diff=lfs merge=lfs -text
Data/**/*.TIF filter=lfs diff=lfs merge=lfs -text
Data/**/*.tiff filter=lfs diff=lfs merge=lfs -text
Data/**/*.tts filter=lfs diff=lfs merge=lfs -text
# WARNING!
# OTB uses Git-LFS to store the (large) tests data.
# Git-LFS is mostly transparent for the user and recent versions
# are able to use Git-LFS quite efficiently.
# But Git fails to manage efficiently numerous LFS data.
# We have to use directly git-lfs wrapping commands to have an
# efficient cloning step.
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)
variables:
BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
DOCKER_DRIVER: overlay2
GIT_DEPTH: "3"
# Disable automatic checkout to let us fetch LFS before
GIT_CHECKOUT: "false"
# The fetch strategy fails with LFS and GitLab
GIT_STRATEGY: "clone"
stages:
- precheck
- prepare
- build
- report
- deploy
.general:
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
.common:
extends: .general
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 -u CI/cdash_handler.py
.common-build:
extends: .common
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: build
artifacts:
when: always
expire_in: 24 hrs
paths:
- build/*/*.log #CMake log
- log/*.txt # Others
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
- build_packages/OTB-*.run
- build_packages/OTB-*.zip
- build/compile_commands.json
- build/ctest_report.xml
- build/cppcheck_report.xml
- build/coverage_report.xml
.common-prepare:
extends: .general
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: prepare
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
artifacts:
expire_in: 24 hrs
when: always
paths:
- sb_branch.txt # Needed to checkout correct branch in build step
- build/*/*/*/*.log # Superbuild log
#-------------------------- precheck job ---------------------------------------
fast-build:
extends: .common
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
#------------------------- prepare & build jobs --------------------------------
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
dependencies: []
## Ubuntu superbuild
ubuntu-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
ubuntu-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
dependencies:
- ubuntu-xdk-prepare
## CentOS superbuild
centos-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
centos-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies:
- centos-xdk-prepare
## MacOS superbuild
macos-xdk-prepare:
extends: .common-prepare
tags:
- macos
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
macos-xdk-build:
tags:
- macos
extends: .common-build
script:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies:
- macos-xdk-prepare
## Windows
.windows-prepare:
extends: .common-prepare
before_script:
# This override the previous before_script
- set GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q %CI_COMMIT_SHA%
- set GIT_LFS_SKIP_SMUDGE=0
.windows-build:
extends: .common-build
before_script:
- git lfs fetch origin %CI_COMMIT_SHA%
- git checkout -f -q %CI_COMMIT_SHA%
# - Win10
windows-10-prepare:
extends: .windows-prepare
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 xdk 10
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-10-build:
extends: .windows-build
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 otb 10
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-10-prepare
# - Win8.1
windows-8-prepare:
extends: .windows-prepare
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 xdk 8.1
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-8-build:
extends: .windows-build
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 otb 8.1
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-8-prepare
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
ctest -V -S CI/main_qa.cmake
-DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
-DQA:BOOL=ON
- ./CI/otb_coverage.sh
- saxon-xslt -o build/ctest_report.xml
build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
CI/ctest2junit.xsl
dependencies:
- ubuntu-xdk-prepare
ubuntu-xdk-qa-static-analysis:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
script:
- find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
grep -vE '^Modules/ThirdParty/' |
sort -u > header_directories
- cppcheck -j 8 -q --xml --xml-version=2 --enable=all
--language=c++ --std=c++14 --platform=unix64
--includes-file=header_directories
-i Modules/ThirdParty Modules
2> build/cppcheck_report.xml
after_script: []
dependencies:
- ubuntu-xdk-prepare
ubuntu-xdk-report:
extends: .common
variables:
GIT_DEPTH: ""
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
stage: report
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
script:
- sonar-scanner -Dproject.settings=sonar-project.properties
-Dsonar.host.url=https://sonar.orfeo-toolbox.org
-Dsonar.login=$SONAR_OTB_TOKEN
`test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
after_script: []
dependencies:
- ubuntu-xdk-qa-code-coverage
- ubuntu-xdk-qa-static-analysis
#---------------------------- Deploy job ---------------------------------------
deploy:
tags:
- deploy
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
- develop@orfeotoolbox/otb
- /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
script:
- ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
dependencies:
- ubuntu-xdk-build
- centos-xdk-build
- macos-xdk-build
- windows-8-build
- windows-10-build
release-container:
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
refs:
- tags@orfeotoolbox/otb
variables:
- $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
script:
- curl --request POST
--form token=$K8S_SECRET_RELEASE
--form ref=master
--form variables[OTB_TAG]=$CI_COMMIT_TAG
https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline
......@@ -36,10 +36,8 @@ Once all blocking issues are closed, and the previous steps are done:
* [ ] Software Guide
* [ ] Cookbook
* [ ] Doxygen
* [ ] Application online documentation
* [ ] WordPress page "Home" and "Download" pages
* [ ] Upload OTB source archive to [Zenodo](https://zenodo.org/) to create a unique Digital Object Identifier (DOI)
* [ ] Update OTB-Data-Examples.tgz on orfeo-toolbox (packages)
* [ ] Send email to mailing list to announce the release
* [ ] Release announcement on the blog
* [ ] Announcement on social networks (twitter, google+)
......
......@@ -45,3 +45,4 @@ The copyright owner is *COPYRIGHT OWNER (OR OWNER'S AGENT)* and has signed the O
- The feature branch is (reasonably) up-to-date with the base branch
- Dashboard is green
- Copyright owner has signed the ORFEO ToolBox Contributor License Agreement
- Optionally, run `git diff develop... -U0 --no-color | clang-format-diff.py -p1 -i` on latest changes and commit
FROM ubuntu:18.04
MAINTAINER Sebastien Dinot <sebastien.dinot@c-s.fr>
ENV DEBIAN_FRONTEND noninteractive
RUN echo "Europe/Paris" > /etc/timezone
# ----------------------------------------------------------------------------
# First stage : install tools (they rarely evolve)
# ----------------------------------------------------------------------------
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
make \
cmake \
ninja-build \
g++ \
gcc \
clang \
clang-tidy \
clang-format \
ccache \
git \
libtool \
swig \
xvfb \
&& rm -rf /var/lib/apt/lists/*
# ----------------------------------------------------------------------------
# Second stage : dependencies (they evolve more often)
# ----------------------------------------------------------------------------
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
freeglut3-dev \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-graph-dev \
libboost-program-options-dev \
libboost-system-dev \
libboost-thread-dev \
libcurl4-gnutls-dev \
libexpat1-dev \
libfftw3-dev \
libgdal-dev \
libgeotiff-dev \
libglew-dev \
libglfw3-dev \
libgsl-dev \
libinsighttoolkit4-dev \
libkml-dev \
libmuparser-dev \
libmuparserx-dev \
libopencv-core-dev \
libopencv-ml-dev \
libopenmpi-dev \
libopenthreads-dev \
libossim-dev \
libpng-dev \
libqt5opengl5-dev \
libqwt-qt5-dev \
libsvm-dev \
libtinyxml-dev \
qtbase5-dev \
qttools5-dev \
default-jdk \
python-dev \
python-numpy \
python-gdal \
python3-dev \
python3-numpy \
python3-gdal \
&& rm -rf /var/lib/apt/lists/*
ENV PATH /usr/lib/ccache:$PATH
ENV OTB_USE_SHARK OFF
FROM ubuntu:18.04 as shark-provider
MAINTAINER OrfeoToolbox Core Team
ENV DEBIAN_FRONTEND noninteractive
# ----------------------------------------------------------------------------
# First stage : install tools
# (based on https://github.com/Shark-ML/Shark/blob/master/.travis.yml)
# ----------------------------------------------------------------------------
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
ca-certificates \
g++ \
make \
cmake \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-graph-dev \
libboost-program-options-dev \
libboost-system-dev \
libboost-thread-dev \
libcurl4-gnutls-dev \
curl \
&& rm -rf /var/lib/apt/lists/*
RUN cd \tmp \
&& curl -o shark.tar.gz https://codeload.github.com/Shark-ML/Shark/tar.gz/v3.1.4 \
&& tar xzf shark.tar.gz \
&& cd Shark-3.1.4\
&& mkdir build \
&& cd build \
&& cmake -DBUILD_EXAMPLES:BOOL=OFF \
-DBUILD_TESTING:BOOL=OFF \
-DENABLE_HDF5:BOOL=OFF \
-DBUILD_SHARED_LIBS=ON \
-DENABLE_CBLAS:BOOL=OFF \
-DENABLE_OPENMP:BOOL=OFF \
../. \
&& make install
FROM ubuntu:18.04
MAINTAINER Sebastien Dinot <sebastien.dinot@c-s.fr>
ENV DEBIAN_FRONTEND noninteractive
RUN echo "Europe/Paris" > /etc/timezone
# ----------------------------------------------------------------------------
# First stage : install tools (they rarely evolve)
# ----------------------------------------------------------------------------
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
make \
cmake \
ninja-build \
g++ \
gcc \
clang \
clang-tidy \
clang-format \
ccache \
git \
libtool \
swig \
xvfb \
&& rm -rf /var/lib/apt/lists/*
# ----------------------------------------------------------------------------
# Second stage : dependencies (they evolve more often)
# ----------------------------------------------------------------------------
RUN apt-get update -y \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends \
freeglut3-dev \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-graph-dev \
libboost-program-options-dev \
libboost-system-dev \
libboost-thread-dev \
libcurl4-gnutls-dev \
libexpat1-dev \
libfftw3-dev \
libgdal-dev \
libgeotiff-dev \
libglew-dev \
libglfw3-dev \
libgsl-dev \
libinsighttoolkit4-dev \
libkml-dev \
libmuparser-dev \
libmuparserx-dev \
libopencv-core-dev \
libopencv-ml-dev \
libopenmpi-dev \
libopenthreads-dev \
libossim-dev \
libpng-dev \
libqt5opengl5-dev \
libqwt-qt5-dev \
libsvm-dev \
libtinyxml-dev \
qtbase5-dev \
qttools5-dev \
default-jdk \
python-dev \
python-numpy \
python-gdal \
python3-dev \
python3-numpy \
python3-gdal \
&& rm -rf /var/lib/apt/lists/*
COPY --from=shark-provider /usr/local/include/shark/* /usr/include/shark/
COPY --from=shark-provider /usr/local/lib/libshark* /usr/lib/
COPY --from=shark-provider /usr/local/bin/SharkVersion /usr/bin/
COPY --from=shark-provider /usr/local/lib/cmake/ /usr/lib/cmake/
ENV PATH /usr/lib/ccache:$PATH
ENV OTB_USE_SHARK ON
# CI
This folder contains files for the continuous integration.
## Docker files
The docker files are:
* Dockerfile.nat: native environment to test OTB. Shark is not on this one as the version OTB is using is not packaged (since 6.7)
* Dockerfile.shark.1.3.4: native environment and Shark version 3.1.4.
## CMake files
The docker files are:
* main_ci.cmake: main script that will launch configure, build and soon test. It aims at configuring Ctest before running the different jobs (site name, build command...)
* configure_option.cmake: script that gathers all options needed by OTB.
## Yaml file
The last file needed for the CI is the *.gitlab-ci.yml* it aims at interfacing with GitLab. Here is the basic structure:
```
Job-name:
stage: job-type
image: docker_image_that_will_be_used
script:
- script that will be launch inside the docker
```
# CDash interaction
Results of build can be seen in CDash on the track "Experimental". The site is the image docker that run the build, and the build name should be the branch name that is tested.
\ No newline at end of file
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import os.path
import urllib.request
import urllib.parse