Merge branch 'ctest-ci' into 'develop'

New CI for OTB

See merge request orfeotoolbox/otb!374

.gitlab-ci.yml

+variables:
+  DOCKER_DRIVER: overlay2
+
+stages:
+  - build
+
+.general:
+  only: [merge_requests]
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+  # variables:
+  #   CI_DEBUG_TRACE: "true"
+
+native-build:
+  extends: .general
+  stage: build
+  image: registry.orfeo-toolbox.org/orfeotoolbox/otb:ubuntu-18.04-gcc
+  script:
+    - ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
+
+shark-build:
+  extends: .general
+  stage: build
+  image: registry.orfeo-toolbox.org/orfeotoolbox/otb:ubuntu-18.04-gcc-shark-3.1.4
+  script:
+    - ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc-shark-3.1.4
\ No newline at end of file

CI/Dockerfile.nat

+FROM ubuntu:18.04
+MAINTAINER Sebastien Dinot <sebastien.dinot@c-s.fr>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN echo "Europe/Paris" > /etc/timezone
+
+# ----------------------------------------------------------------------------
+# First stage : install tools (they rarely evolve)
+# ----------------------------------------------------------------------------
+RUN apt-get update -y \
+ && apt-get upgrade -y \
+ && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        make \
+        cmake \
+        ninja-build \
+        g++ \
+        gcc \
+        clang \
+        clang-tidy \
+        clang-format \
+        ccache \
+        git \
+        libtool \
+        swig \
+        xvfb \
+ && rm -rf /var/lib/apt/lists/*
+
+# ----------------------------------------------------------------------------
+# Second stage : dependencies (they evolve more often)
+# ----------------------------------------------------------------------------
+RUN apt-get update -y \
+ && apt-get upgrade -y \
+ && apt-get install -y --no-install-recommends \
+        freeglut3-dev \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-graph-dev \
+        libboost-program-options-dev \
+        libboost-system-dev \
+        libboost-thread-dev \
+        libcurl4-gnutls-dev \
+        libexpat1-dev \
+        libfftw3-dev \
+        libgdal-dev \
+        libgeotiff-dev \
+        libglew-dev \
+        libglfw3-dev \
+        libgsl-dev \
+        libinsighttoolkit4-dev \
+        libkml-dev \
+        libmuparser-dev \
+        libmuparserx-dev \
+        libopencv-core-dev \
+        libopencv-ml-dev \
+        libopenmpi-dev \
+        libopenthreads-dev \
+        libossim-dev \
+        libpng-dev \
+        libqt5opengl5-dev \
+        libqwt-qt5-dev \
+        libsvm-dev \
+        libtinyxml-dev \
+        qtbase5-dev \
+        qttools5-dev \
+        default-jdk \
+        python-dev \
+        python-numpy \
+        python-gdal \
+        python3-dev \
+        python3-numpy \
+        python3-gdal \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH /usr/lib/ccache:$PATH
+ENV OTB_USE_SHARK OFF

CI/Dockerfile.shark3.1.4

+FROM ubuntu:18.04 as shark-provider
+MAINTAINER OrfeoToolbox Core Team
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# ----------------------------------------------------------------------------
+# First stage : install tools 
+# (based on https://github.com/Shark-ML/Shark/blob/master/.travis.yml)
+# ----------------------------------------------------------------------------
+
+RUN apt-get update -y \
+  && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        g++ \
+        make \
+        cmake \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-graph-dev \
+        libboost-program-options-dev \
+        libboost-system-dev \
+        libboost-thread-dev \
+        libcurl4-gnutls-dev \
+        curl \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN cd \tmp \
+  && curl -o shark.tar.gz https://codeload.github.com/Shark-ML/Shark/tar.gz/v3.1.4 \
+  && tar xzf shark.tar.gz \
+  && cd Shark-3.1.4\
+  && mkdir build \
+  && cd build \
+  && cmake  -DBUILD_EXAMPLES:BOOL=OFF \
+            -DBUILD_TESTING:BOOL=OFF \
+            -DENABLE_HDF5:BOOL=OFF \
+            -DBUILD_SHARED_LIBS=ON \
+            -DENABLE_CBLAS:BOOL=OFF \
+            -DENABLE_OPENMP:BOOL=OFF \
+            ../. \
+  && make install
+
+FROM ubuntu:18.04
+MAINTAINER Sebastien Dinot <sebastien.dinot@c-s.fr>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN echo "Europe/Paris" > /etc/timezone
+
+# ----------------------------------------------------------------------------
+# First stage : install tools (they rarely evolve)
+# ----------------------------------------------------------------------------
+RUN apt-get update -y \
+ && apt-get upgrade -y \
+ && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        make \
+        cmake \
+        ninja-build \
+        g++ \
+        gcc \
+        clang \
+        clang-tidy \
+        clang-format \
+        ccache \
+        git \
+        libtool \
+        swig \
+        xvfb \
+ && rm -rf /var/lib/apt/lists/*
+
+# ----------------------------------------------------------------------------
+# Second stage : dependencies (they evolve more often)
+# ----------------------------------------------------------------------------
+
+RUN apt-get update -y \
+ && apt-get upgrade -y \
+ && apt-get install -y --no-install-recommends \
+        freeglut3-dev \
+        libboost-date-time-dev \
+        libboost-filesystem-dev \
+        libboost-graph-dev \
+        libboost-program-options-dev \
+        libboost-system-dev \
+        libboost-thread-dev \
+        libcurl4-gnutls-dev \
+        libexpat1-dev \
+        libfftw3-dev \
+        libgdal-dev \
+        libgeotiff-dev \
+        libglew-dev \
+        libglfw3-dev \
+        libgsl-dev \
+        libinsighttoolkit4-dev \
+        libkml-dev \
+        libmuparser-dev \
+        libmuparserx-dev \
+        libopencv-core-dev \
+        libopencv-ml-dev \
+        libopenmpi-dev \
+        libopenthreads-dev \
+        libossim-dev \
+        libpng-dev \
+        libqt5opengl5-dev \
+        libqwt-qt5-dev \
+        libsvm-dev \
+        libtinyxml-dev \
+        qtbase5-dev \
+        qttools5-dev \
+        default-jdk \
+        python-dev \
+        python-numpy \
+        python-gdal \
+        python3-dev \
+        python3-numpy \
+        python3-gdal \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY --from=shark-provider /usr/local/include/shark/* /usr/include/shark/
+COPY --from=shark-provider /usr/local/lib/libshark* /usr/lib/
+COPY --from=shark-provider /usr/local/bin/SharkVersion /usr/bin/
+COPY --from=shark-provider /usr/local/lib/cmake/ /usr/lib/cmake/
+
+ENV PATH /usr/lib/ccache:$PATH
+ENV OTB_USE_SHARK ON

CI/README.md

+# CI
+This folder contains files for the continuous integration.
+
+## Docker files
+The docker files are: 
+
+ * Dockerfile.nat: native environment to test OTB. Shark is not on this one as the version OTB is using is not packaged (since 6.7)
+ * Dockerfile.shark.1.3.4: native environment and Shark version 3.1.4.
+
+## CMake files
+The docker files are:
+
+ * main_ci.cmake: main script that will launch configure, build and soon test. It aims at configuring Ctest before running the different jobs (site name, build command...)
+ * configure_option.cmake: script that gathers all options needed by OTB.
+
+## Yaml file
+The last file needed for the CI is the *.gitlab-ci.yml* it aims at interfacing with GitLab. Here is the basic structure: 
+```
+Job-name:
+  stage: job-type
+  image: docker_image_that_will_be_used
+  script:
+    - script that will be launch inside the docker
+```
+
+# CDash interaction
+Results of build can be seen in CDash on the track "Experimental". The site is the image docker that run the build, and the build name should be the branch name that is tested.
\ No newline at end of file

CI/configure_option.cmake

+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script is a prototype for the future CI, it may evolve rapidly in a near future
+#This file set the following variable : 
+# * otb_build_project_option
+# * otb_use_option
+# * otb_wrap_option
+# * CONFIGURE_OPTIONS
+
+set (otb_build_project_option 
+"BUILD_COOKBOOK:BOOL=OFF
+BUILD_EXAMPLES:BOOL=ON
+BUILD_SHARED_LIBS:BOOL=ON
+BUILD_TESTING:BOOL=ON")
+
+set (otb_use_option
+"OTB_USE_6S:BOOL=ON
+OTB_USE_CURL:BOOL=ON
+OTB_USE_GLEW:BOOL=ON
+OTB_USE_GLFW:BOOL=ON
+OTB_USE_GLUT:BOOL=ON
+OTB_USE_GSL:BOOL=ON
+OTB_USE_LIBKML:BOOL=ON
+OTB_USE_LIBSVM:BOOL=ON
+OTB_USE_MPI:BOOL=ON
+OTB_USE_MUPARSER:BOOL=ON
+OTB_USE_MUPARSERX:BOOL=ON
+OTB_USE_OPENCV:BOOL=ON
+OTB_USE_OPENGL:BOOL=ON
+OTB_USE_OPENMP:BOOL=ON
+OTB_USE_QT:BOOL=ON
+OTB_USE_QWT:BOOL=ON
+OTB_USE_SHARK:BOOL=$ENV{OTB_USESHARK}
+OTB_USE_SIFTFAST:BOOL=ON
+OTB_USE_SPTW:BOOL=ON
+OTB_USE_SSE_FLAGS:BOOL=ON")
+
+set (otb_wrap_option
+"OTB_WRAP_JAVA:BOOL=ON
+OTB_WRAP_PYTHON:BOOL=ON
+OTB_WRAP_PYTHON3:BOOL=ON")
+
+set (otb_data_option
+"OTB_DATA_ROOT:PATH=${OTB_DATA_ROOT}
+OTB_DATA_USE_LARGEINPUT:BOOL=OFF
+OTB_DATA_LARGEINPUT_ROOT:PATH=${OTB_LARGEINPUT_ROOT}")
+
+set (cmake_configure_option
+"CMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION}
+CMAKE_INSTALL_PREFIX:PATH=${CTEST_INSTALL_DIRECTORY}
+CMAKE_C_FLAGS:STRING=-fPIC -Wall -Wextra
+CMAKE_CXX_FLAGS:STRING=-fPIC -Wall -Wextra -Wno-cpp")
+
+#Transform the previous string in list
+string (REPLACE "\n" ";" otb_build_project_option ${otb_build_project_option})
+string (REPLACE "\n" ";" otb_use_option ${otb_use_option})
+string (REPLACE "\n" ";" otb_wrap_option ${otb_wrap_option})
+string (REPLACE "\n" ";" otb_data_option ${otb_data_option})
+
+set (otb_options "")
+list (APPEND otb_options ${otb_build_project_option})
+list (APPEND otb_options ${otb_use_option})
+list (APPEND otb_options ${otb_wrap_option})
+list (APPEND otb_options ${otb_data_option})
+
+foreach(option ${otb_options})
+  set( CONFIGURE_OPTIONS "${CONFIGURE_OPTIONS}-D${option};")
+endforeach(option)
+
+# This should be removed as soon as possible
+set( CONFIGURE_OPTIONS "${CONFIGURE_OPTIONS}-Dopencv_INCLUDE_DIR:PATH=/usr/include;")

CI/main_ci.cmake

+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script is a prototype for the future CI, it may evolve rapidly in a near future
+get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
+set (ENV{LANG} "C") # Only ascii output
+
+# Build Configuration : Release, Debug..
+set (CTEST_BUILD_CONFIGURATION "Release")
+set (CTEST_CMAKE_GENERATOR "Unix Makefiles")
+
+set (CTEST_BUILD_NAME "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}_to_$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
+set (CTEST_SITE "${IMAGE_NAME}")
+
+# Directory variable
+set (CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}")
+set (CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/")
+set (CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install/")
+set (PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}")
+
+# Ctest command value
+set (CMAKE_COMMAND "cmake")
+set (CTEST_BUILD_FLAGS "-j8") # number of thread used for compilation
+set (CTEST_BUILD_COMMAND "make ${CTEST_BUILD_FLAGS}")
+
+
+
+# Data directory setting
+set (OTB_DATA_ROOT "${OTB_SOURCE_DIR}/otb-data/") # todo
+set (OTB_LARGEINPUT_ROOT "") # todo
+
+
+#The following file set the CONFIGURE_OPTIONS variable
+set (CONFIGURE_OPTIONS  "")
+include ( "${CMAKE_CURRENT_LIST_DIR}/configure_option.cmake" )
+
+# End of configuration
+
+
+ctest_start (Experimental TRACK Experimental)
+
+ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
+    SOURCE "${OTB_SOURCE_DIR}"
+    OPTIONS "${CONFIGURE_OPTIONS}"
+    RETURN_VALUE _configure_rv
+    CAPTURE_CMAKE_ERROR _configure_error)
+
+if ( _configure_rv EQUAL -1 )
+  message("An error occurs during ctest_configure:
+    ${_configure_error}")
+endif()
+
+ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
+            RETURN_VALUE _build_rv]
+            CAPTURE_CMAKE_ERROR _build_error]
+            )
+
+if ( _configure_rv EQUAL -1 )
+  message("An error occurs during ctest_build:
+    ${_build_error}")
+endif()
+
+# ctest_test(PARALLEL_LEVEL 8])
+
+ctest_submit()
\ No newline at end of file