Merge branch 'superbuild_ci-centos-6' into 'develop'

Superbuild CI on Centos 6 See merge request orfeotoolbox/otb!466

Merge branch 'superbuild_ci-centos-6' into 'develop'
44005a23 · Guillaume Pasero · 04207852 · 5bb847b5 · 44005a23 · 44005a23
Commit 44005a23 authored 5 years ago by Guillaume Pasero
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,7 +5,7 @@
 # But Git fails to manage efficiently numerous LFS data.
 # We have to use directly git-lfs wrapping commands to have an
 # efficient cloning step.
-# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of 
+# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
 # GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)

 variables:
@@ -53,17 +53,19 @@ fast-build:
    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
    - ccache -s

-debian-build:
+.common-build:
  extends: .general
  only: [merge_requests]
  stage: build
+
+debian-build:
+  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc

-.common-build:
-  extends: .general
-  stage: build
+ubuntu-llvm:
+  extends: .common-build
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
  script:
    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm
@@ -73,15 +75,10 @@ debian-build:
      - build/Documentation/Cookbook/latex/CookBook-*.pdf
      - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2

-ubuntu-llvm:
-  only: [merge_requests]
-  extends: .common-build
-
-superbuild-prepare:
-  only: [merge_requests]
+.common-prepare:
  extends: .general
+  only: [merge_requests]
  stage: prepare
-  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  before_script:
 # This override the previous before_script
    - git checkout $CI_COMMIT_REF_NAME
@@ -95,8 +92,6 @@ superbuild-prepare:
    - ssh-add <(echo "$K8S_SECRET_SSH")
 # This is for debug, we are checking the owner of the ssh key
    - ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
-  script:
-    - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
  after_script:
    - echo "Nothing to do for after_script"
  artifacts:
@@ -106,12 +101,30 @@ superbuild-prepare:
      - build/*/*/*/*.log
      - sb_branch.txt

-superbuild-build:
-  only: [merge_requests]
-  extends: .general
-  stage: build
+ubuntu-superbuild-prepare:
+  extends: .common-prepare
  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
  script:
-    - ctest -VV -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
+    - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
+
+centos-superbuild-prepare:
+  extends: .common-prepare
+  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
+  script:
+    - ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
+
+ubuntu-superbuild-build:
+  extends: .common-build
+  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
+  script:
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
+  dependencies:
+    - ubuntu-superbuild-prepare
+
+centos-superbuild-build:
+  extends: .common-build
+  image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
+  script:
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
  dependencies:
-    - superbuild-prepare
+    - centos-superbuild-prepare
--- a/CI/configure_option.cmake
+++ b/CI/configure_option.cmake
@@ -19,13 +19,13 @@
 #

 # This script is a prototype for the future CI, it may evolve rapidly in a near future
-#This file set the following variable : 
+#This file set the following variable :
 # * otb_build_project_option
 # * otb_use_option
 # * otb_wrap_option
 # * CONFIGURE_OPTIONS

-set (otb_build_project_option 
+set (otb_build_project_option
 "BUILD_COOKBOOK:BOOL=OFF
 BUILD_EXAMPLES:BOOL=ON
 BUILD_SHARED_LIBS:BOOL=ON
@@ -40,7 +40,7 @@ OTB_USE_GLUT:BOOL=ON
 OTB_USE_GSL:BOOL=ON
 OTB_USE_LIBKML:BOOL=ON
 OTB_USE_LIBSVM:BOOL=ON
-OTB_USE_MPI:BOOL=ON
+OTB_USE_MPI:BOOL=OFF
 OTB_USE_MUPARSER:BOOL=ON
 OTB_USE_MUPARSERX:BOOL=ON
 OTB_USE_OPENCV:BOOL=ON

--- a/CI/macros.cmake
+++ b/CI/macros.cmake
+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script is for the superbuild build on the CI platform
+
+# Find the build name and CI profile
+macro( set_dash_build_name )
+  set(ci_profile wip)
+
+  set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
+  set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
+  set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}")
+
+  set(ci_pipeline_id "$ENV{CI_PIPELINE_ID}")
+  set(ci_job_id "$ENV{CI_JOB_ID}")
+
+  set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
+
+  set (CTEST_BUILD_NAME ${ci_short_sha})
+
+  if(ci_mr_source AND ci_mr_target AND ci_mr_iid)
+    set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid}) P-${ci_pipeline_id}")
+    set(ci_profile mr)
+
+  elseif(ci_ref_name)
+    set (CTEST_BUILD_NAME "${ci_ref_name} P-${ci_pipeline_id}")
+
+    if("${ci_ref_name}" STREQUAL "develop")
+      set(ci_profile develop)
+
+    elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$")
+      set(ci_profile release)
+    endif()
+
+  endif()
+
+endmacro()
--- a/CI/main_ci.cmake
+++ b/CI/main_ci.cmake
@@ -18,6 +18,8 @@
 # limitations under the License.
 #

+include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
+
 # This script is a prototype for the future CI, it may evolve rapidly in a near future
 get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 set (ENV{LANG} "C") # Only ascii output
@@ -36,23 +38,7 @@ else()
 endif()

 # Find the build name and CI profile
-set(ci_profile wip)
-set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
-set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
-set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}")
-set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
-set (CTEST_BUILD_NAME ${ci_short_sha})
-if(ci_mr_source AND ci_mr_target AND ci_mr_iid)
-  set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid})")
-  set(ci_profile mr)
-elseif(ci_ref_name)
-  set (CTEST_BUILD_NAME "${ci_ref_name}")
-  if("${ci_ref_name}" STREQUAL "develop")
-    set(ci_profile develop)
-  elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$")
-    set(ci_profile release)
-  endif()
-endif()
+set_dash_build_name()

 # set pipelines to enable documentation
 set(ci_cookbook_profiles mr develop release)
@@ -109,7 +95,7 @@ find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
 # End of configuration


-ctest_start (Experimental TRACK Experimental)
+ctest_start (Experimental TRACK CI_Build)

 ctest_update()


--- a/CI/main_superbuild.cmake
+++ b/CI/main_superbuild.cmake
@@ -18,6 +18,8 @@
 # limitations under the License.
 #

+include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
+
 # This script is a prototype for the future CI, it may evolve rapidly in a near future
 set (ENV{LANG} "C") # Only ascii output
 get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
@@ -44,7 +46,12 @@ file(READ "${OTB_SOURCE_DIR}/sb_branch.txt" BRANCH_NAME)
 ###############################################################################
 set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact.git")
 # set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}")
-set( GIT "git" )
+
+# Look for a GIT command-line client.
+find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
+
+# FIXME: Replace ${GIT} variable with $[CTEST_GIT_COMMAND}"
+set( GIT "${CTEST_GIT_COMMAND}" )

 execute_process(
  COMMAND ${GIT} "clone" "${REMOTE}" "--branch" "${BRANCH_NAME}"
@@ -69,7 +76,7 @@ endif()

 set (CMAKE_COMMAND "cmake")
 execute_process(
-  COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf" 
+  COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf"
  "${CI_PROJ_DIR}/superbuild-artifact/SuperBuild_Install.tar"
  WORKING_DIRECTORY ${CI_ROOT_DIR}
  )
@@ -92,10 +99,12 @@ endif()

 set ( CTEST_BUILD_CONFIGURATION "Release" )
 set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
-set ( CTEST_BUILD_FLAGS "-j16" )
-set ( CTEST_BUILD_NAME "Superbuild_Build_Otb" )
+set ( CTEST_BUILD_FLAGS "-j8" )
 set ( CTEST_SITE "${IMAGE_NAME}" )

+# Find the build name and CI profile
+set_dash_build_name()
+
 # Directory variable
 set ( CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}" )
 set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
@@ -105,25 +114,30 @@ set ( PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}" )
 set (CONFIGURE_OPTIONS  "")
 include ( "${CMAKE_CURRENT_LIST_DIR}/configure_option.cmake" )
 # SuperBuild case : one more configure option
-set ( CONFIGURE_OPTIONS  
+set ( CONFIGURE_OPTIONS
  "${CONFIGURE_OPTIONS}-DCMAKE_PREFIX_PATH=${XDK_PATH};")

 # Hack because there is no more superbuild available (LIBKML)
-set ( CONFIGURE_OPTIONS  
+set ( CONFIGURE_OPTIONS
  "${CONFIGURE_OPTIONS}-DOTB_USE_LIBKML:BOOL=OFF;" )

 # FIX ME this part might platform dependent
 set( GDAL_DATA "${XDK_PATH}/share/gdal" )
 set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
 set( PROJ_LIB "${XDK_PATH}/share" )
-set( CTEST_ENVIRONMENT 
+set( CTEST_ENVIRONMENT
 "PATH=${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}
 GDAL_DATA= GDAL_DATA
 GEOTIFF_CSV= GEOTIFF_CSV
 PROJ_LIB= PROJ_LIB
 ")

-ctest_start (Experimental TRACK Experimental)
+# Sources are already checked out : do nothing for update
+set(CTEST_GIT_UPDATE_CUSTOM echo No update)
+
+ctest_start (Experimental TRACK CI_Build)
+
+ctest_update()

 ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
    SOURCE "${OTB_SOURCE_DIR}"
@@ -134,7 +148,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"

 if ( NOT _configure_rv EQUAL 0 )
  ctest_submit()
-  message( SEND_ERROR "An error occurs during ctest_configure.")
+  message( FATAL_ERROR "An error occurs during ctest_configure.")
 endif()

 ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
@@ -143,19 +157,17 @@ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
            )

 if ( NOT _build_rv EQUAL 0 )
-  ctest_submit()
  message( SEND_ERROR "An error occurs during ctest_build.")
 endif()

 # Uncomment when ready for test
-# ctest_test(PARALLEL_LEVEL 8
-#            RETURN_VALUE _test_rv
-#            CAPTURE_CMAKE_ERROR _test_error
-#            )
-
-# if ( NOT _test_rv EQUAL 0 )
-#   ctest_submit()
-#   message( SEND_ERROR "An error occurs during ctest_test.")
-# endif()
+ctest_test(PARALLEL_LEVEL 8
+           RETURN_VALUE _test_rv
+           CAPTURE_CMAKE_ERROR _test_error
+           )
+
+if ( NOT _test_rv EQUAL 0 )
+  message( WARNING "Some tests have failed.")
+endif()

 ctest_submit()
--- a/CI/prepare_superbuild.cmake
+++ b/CI/prepare_superbuild.cmake
@@ -19,6 +19,8 @@
 #
 # This script is for the superbuild build on the CI platform

+include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
+
 set (ENV{LANG} "C") # Only ascii output
 get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
 get_filename_component(CI_PROJ_DIR ${OTB_SOURCE_DIR} DIRECTORY)
@@ -39,7 +41,10 @@ set ( PROJECT_SOURCE_DIR "${SUPERBUILD_SOURCE_DIR}" )
 set ( CTEST_SOURCE_DIRECTORY "${SUPERBUILD_SOURCE_DIR}" )
 set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
 set ( CTEST_SITE "${IMAGE_NAME}" )
-set ( CTEST_BUILD_NAME "Superbuild_Build_Depends" ) # FIXME
+
+
+# Find the build name and CI profile
+set_dash_build_name()

 # We need a directory independent from user
 # in CI the architecture is /builds/user/otb
@@ -50,7 +55,7 @@ set ( CTEST_BUILD_NAME "Superbuild_Build_Depends" ) # FIXME
 set (CTEST_INSTALL_DIRECTORY "${CI_ROOT_DIR}/xdk/")

 # HACK
-# This is needed because when using return() function ctest is trying 
+# This is needed because when using return() function ctest is trying
 # to run the CTEST_COMMAND. And we need it to not produce an error
 set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
 set (CMAKE_COMMAND "cmake")
@@ -61,12 +66,23 @@ set (CMAKE_COMMAND "cmake")
 ########################################################################
 ########################################################################

-ctest_start (Experimental TRACK Experimental)
+# Look for a GIT command-line client.
+find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
+
+# FIXME: Replace ${GIT} variable with $[CTEST_GIT_COMMAND}"
+set( GIT "${CTEST_GIT_COMMAND}" )
+
+# Sources are already checked out : do nothing for update
+set(CTEST_GIT_UPDATE_CUSTOM echo No update)
+
+ctest_start (Experimental TRACK CI_Prepare)
+
+ctest_update( SOURCE "${OTB_SOURCE_DIR}" )

 set(CTEST_BUILD_FLAGS "-j16")

 set ( SB_CONFIGURE_OPTIONS "")
-include( "${CMAKE_CURRENT_LIST_DIR}/sb_configure_options.cmake" )
+include( "${CMAKE_CURRENT_LIST_DIR}/../SuperBuild/CI/configure_options.cmake" )

 ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
    SOURCE "${SUPERBUILD_SOURCE_DIR}"
@@ -117,9 +133,9 @@ file ( WRITE "${OTB_SOURCE_DIR}/sb_branch.txt" "${IMAGE_NAME}/${SB_MD5}")
 message( "Checking out git for existence of archive")
 set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact/")
 set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}")
-set( GIT "git" )
+
 execute_process(
-  COMMAND ${GIT} "ls-remote" "${REMOTE}" "${BRANCH_NAME}" 
+  COMMAND ${GIT} "ls-remote" "${REMOTE}" "${BRANCH_NAME}"
  OUTPUT_VARIABLE IS_SB_BUILD
  )
 if ( IS_SB_BUILD )
@@ -162,17 +178,17 @@ ctest_submit()
 # The image used will be passed to this script.
 # TODO verify that images does not have forbidden char in there name
 # TODO right now we rely on ctest_build to know whether there has been an error
-# in build, whereas SuperBuild does not necessarily return an error if something 
+# in build, whereas SuperBuild does not necessarily return an error if something
 # goes wrong
 set ( SB_ARTIFACT_GIT "${CI_PROJ_DIR}/superbuild-artifact" )

-# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are 
+# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are
 # different because one is https and one is ssh. Both should be ssh.
 set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git")
 # We clone master to have a basic configuration, mainly a correct .gitattribute
 # git clone $REMOTE --branch master --depth 1 superbuild-artifact
 execute_process(
-  COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}" 
+  COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}"
  "--branch" "master" "--depth" "1" "superbuild-artifact"
  WORKING_DIRECTORY "${CI_PROJ_DIR}"
  )
@@ -181,7 +197,7 @@ execute_process(
 # StrictHostKeyChecking so we don't have to add the host as a known key
 # -F /dev/null so the agent is not taking a default file ~/.ssh/..
 execute_process(
-  COMMAND ${GIT} "config" "core.sshCommand" 
+  COMMAND ${GIT} "config" "core.sshCommand"
  "ssh -o StrictHostKeyChecking=no -F /dev/null"
  WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
  RESULT_VARIABLE ssh_res
@@ -248,13 +264,13 @@ set ( SB_TAR_NAME "SuperBuild_Install.tar" )
 # We need to create tar in its directory to avoid weird name in file
 # "tar: Removing leading `../../' from member names"
 # WARNING
-# We are creating a tar containing xdk/.., so when extracting the archive in 
+# We are creating a tar containing xdk/.., so when extracting the archive in
 # an other environment the output file will be xdk... Obvious isn't it?
 # Well... Not for everyone...
-# May be for easier maintainability the tar name should be the same as the 
+# May be for easier maintainability the tar name should be the same as the
 # file inside.
 execute_process(
-  COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_TAR_NAME}" 
+  COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_TAR_NAME}"
  -- "${CTEST_INSTALL_DIRECTORY}"
  WORKING_DIRECTORY ${CI_ROOT_DIR}
  )
@@ -295,7 +311,7 @@ endif()
 # In our case if toto is deploying a key in superbuild-artifact repo
 # the the mail will be toto's
 execute_process(
-  COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\"" 
+  COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\""
  "-m" "\"New Superbuild for ${SB_MD5} on ${IMAGE_NAME}\""
  WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
  RESULT_VARIABLE com_res
@@ -342,4 +358,4 @@ if ( DEBUG )
  message( "push_res = ${push_res}" )
  message( "push_out = ${push_out}" )
  message( "push_err = ${push_err}" )
-endif() 
\ No newline at end of file
+endif()
--- a/CI/sb_configure_options.cmake
+++ b/CI/sb_configure_options.cmake
@@ -24,7 +24,8 @@ set ( cmake_configure_option
 CMAKE_INSTALL_PREFIX:PATH=${CTEST_INSTALL_DIRECTORY}")

 set ( temporary_option
-"OTB_USE_LIBKML=OFF")
+"OTB_USE_LIBKML=OFF
+OTB_USE_MPI=OFF")

 set(concat_options
 "${cmake_configure_option}
@@ -36,4 +37,4 @@ string (REPLACE "\n" ";" sb_options ${concat_options})

 foreach(item ${sb_options})
  set( SB_CONFIGURE_OPTIONS "${SB_CONFIGURE_OPTIONS}-D${item};")
-endforeach(item)
\ No newline at end of file
+endforeach(item)
--- a/SuperBuild/CMakeLists.txt
+++ b/SuperBuild/CMakeLists.txt
@@ -18,6 +18,8 @@
 # limitations under the License.
 #

+## COMMMENT to dirty superbuild artifact. TAG: 1.
+
 cmake_minimum_required(VERSION 3.3.0)

 project(OTB-SuperBuild)