Commit 8271c62b authored by Guillaume Pasero's avatar Guillaume Pasero

Merge branch 'superbuild_ci' into 'develop'

Superbuild CI

See merge request !459
parents 3df7fcd2 d8a3f2d0
Pipeline #1032 passed with stage
......@@ -29,6 +29,7 @@ after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH
stages:
- prepare
- build
.general:
......@@ -67,3 +68,41 @@ ubuntu-llvm-wip:
except: [merge_requests]
extends: .common-build
superbuild-prepare:
only: [merge_requests]
extends: .general
stage: prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
before_script:
# This override the previous before_script
- git checkout $CI_COMMIT_REF_NAME
# We are now doing the git-lfs install
# This is done after the checkout so we avoid downloading Data
# But we need it to upload the archive
- git-lfs install
- git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot"
- eval $(ssh-agent -s)
- ssh-add <(echo "$K8S_SECRET_SSH")
# This is for debug, we are checking the owner of the ssh key
- ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
after_script:
- echo "Nothing to do for after_script"
artifacts:
expire_in: 24 hrs
paths:
# This recovers logs from superbuild build
- build/*/*/*/*.log
- sb_branch.txt
superbuild-build:
only: [merge_requests]
extends: .general
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
dependencies:
- superbuild-prepare
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is a prototype for the future CI, it may evolve rapidly in a near future
set (ENV{LANG} "C") # Only ascii output
get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
get_filename_component(CI_PROJ_DIR ${OTB_SOURCE_DIR} DIRECTORY)
get_filename_component(CI_ROOT_DIR ${CI_PROJ_DIR} DIRECTORY)
###########################################################################
###########################################################################
# Download xkd
###########################################################################
###########################################################################
# How to get md5sum:
# * concatenate all source files in one
# * add configure result from previous job ${OTB_SOURCE_DIR}/build/CMakeCache.txt
###########################
file(READ "${OTB_SOURCE_DIR}/sb_branch.txt" BRANCH_NAME)
####################################
# git clone $REPOSITORY_URL --branch $BRANCH_NAME --depth 1 superbuild-artifact
###############################################################################
set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact.git")
# set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}")
set( GIT "git" )
execute_process(
COMMAND ${GIT} "clone" "${REMOTE}" "--branch" "${BRANCH_NAME}"
"--depth" "1" "superbuild-artifact"
WORKING_DIRECTORY ${CI_PROJ_DIR}
RESULT_VARIABLE clone_res
OUTPUT_VARIABLE clone_out
ERROR_VARIABLE clone_err
)
if ( DEBUG )
message( "Clone")
message( "clone_res = ${clone_res}" )
message( "clone_out = ${clone_out}" )
message( "clone_err = ${clone_err}" )
endif()
if (clone_res)
message( SEND_ERROR "Problem in retreiving the archive")
return()
endif()
set (CMAKE_COMMAND "cmake")
execute_process(
COMMAND ${CMAKE_COMMAND} "-E" "tar" "xf"
"${CI_PROJ_DIR}/superbuild-artifact/SuperBuild_Install.tar"
WORKING_DIRECTORY ${CI_ROOT_DIR}
)
set( XDK_PATH "${CI_ROOT_DIR}/xdk")
if ( DEBUG )
if ( EXISTS "${XDK_PATH}")
message("Tar file exists at ${XDK_PATH}")
else()
message("Something went wrong no tar file in ${XDK_PATH}")
endif()
endif()
###########################################################################
###########################################################################
# Building OTB
###########################################################################
###########################################################################
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j16" )
set ( CTEST_BUILD_NAME "Superbuild_Build_Otb" )
set ( CTEST_SITE "${IMAGE_NAME}" )
# Directory variable
set ( CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}" )
set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
set ( CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install/" )
set ( PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}" )
set (CONFIGURE_OPTIONS "")
include ( "${CMAKE_CURRENT_LIST_DIR}/configure_option.cmake" )
# SuperBuild case : one more configure option
set ( CONFIGURE_OPTIONS
"${CONFIGURE_OPTIONS}-DCMAKE_PREFIX_PATH=${XDK_PATH};")
# Hack because there is no more superbuild available (LIBKML)
set ( CONFIGURE_OPTIONS
"${CONFIGURE_OPTIONS}-DOTB_USE_LIBKML:BOOL=OFF;" )
# FIX ME this part might platform dependent
set( GDAL_DATA "${XDK_PATH}/share/gdal" )
set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
set( PROJ_LIB "${XDK_PATH}/share" )
set( CTEST_ENVIRONMENT
"PATH=${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}
GDAL_DATA= GDAL_DATA
GEOTIFF_CSV= GEOTIFF_CSV
PROJ_LIB= PROJ_LIB
")
ctest_start (Experimental TRACK Experimental)
ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
SOURCE "${OTB_SOURCE_DIR}"
OPTIONS "${CONFIGURE_OPTIONS}"
RETURN_VALUE _configure_rv
CAPTURE_CMAKE_ERROR _configure_error
)
if ( NOT _configure_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_configure.")
endif()
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
RETURN_VALUE _build_rv
CAPTURE_CMAKE_ERROR _build_error
)
if ( NOT _build_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_build.")
endif()
# Uncomment when ready for test
# ctest_test(PARALLEL_LEVEL 8
# RETURN_VALUE _test_rv
# CAPTURE_CMAKE_ERROR _test_error
# )
# if ( NOT _test_rv EQUAL 0 )
# ctest_submit()
# message( SEND_ERROR "An error occurs during ctest_test.")
# endif()
ctest_submit()
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is for the superbuild build on the CI platform
set (ENV{LANG} "C") # Only ascii output
get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
get_filename_component(CI_PROJ_DIR ${OTB_SOURCE_DIR} DIRECTORY)
get_filename_component(CI_ROOT_DIR ${CI_PROJ_DIR} DIRECTORY)
# In GitLab we have :
# OTB_SOURCE_DIR=/builds/{project_dir}/otb
# CI_PROJ_DIR=/builds/{project_dir}
# CI_ROOT_DIR=/builds
set ( DEBUG "1" )
set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" )
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( PROJECT_SOURCE_DIR "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_SOURCE_DIRECTORY "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
set ( CTEST_SITE "${IMAGE_NAME}" )
set ( CTEST_BUILD_NAME "Superbuild_Build_Depends" ) # FIXME
# We need a directory independent from user
# in CI the architecture is /builds/user/otb
# So we will go in /builds/
# This is platform dependent, and the next step (build) also
# depends on that, as some paths are hardcoded
# This can be fixed with a packaging of OTB_DEPENDS
set (CTEST_INSTALL_DIRECTORY "${CI_ROOT_DIR}/xdk/")
# HACK
# This is needed because when using return() function ctest is trying
# to run the CTEST_COMMAND. And we need it to not produce an error
set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
set (CMAKE_COMMAND "cmake")
########################################################################
########################################################################
# Build process
########################################################################
########################################################################
ctest_start (Experimental TRACK Experimental)
set(CTEST_BUILD_FLAGS "-j16")
set ( SB_CONFIGURE_OPTIONS "")
include( "${CMAKE_CURRENT_LIST_DIR}/sb_configure_options.cmake" )
ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
SOURCE "${SUPERBUILD_SOURCE_DIR}"
OPTIONS "${SB_CONFIGURE_OPTIONS}"
RETURN_VALUE _configure_rv
CAPTURE_CMAKE_ERROR _configure_error
)
if ( NOT _configure_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_configure. Dependencies might be buggy.")
return()
endif()
########################################################################
########################################################################
# Check process
########################################################################
########################################################################
# Once that we have configure our build we can check if it exists a
# corresponding SB on superbuild-artifact
# How to get md5sum:
# * concatenate all source files in one
# * add configure result : CMakeCache.txt
####################################
file( GLOB_RECURSE sb_file_list "${OTB_SOURCE_DIR}/SuperBuild/*")
set( SB_TXT "${OTB_SOURCE_DIR}/full_sb.txt")
foreach(sb_file ${sb_file_list})
file(READ ${sb_file} CONTENTS)
file(APPEND ${SB_TXT} "${sb_file}${CONTENTS}")
endforeach(sb_file)
file(READ "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" CMAKE_ORIG)
string(REPLACE "${CI_PROJ_DIR}" "" CMAKE_UNIFIED ${CMAKE_ORIG})
file(APPEND ${SB_TXT} "CMakeCache.txt${CMAKE_UNIFIED}")
file ( MD5 "${SB_TXT}" SB_MD5)
message ( "SB_MD5 = ${SB_MD5}" )
file (REMOVE ${SB_TXT})
####################################
# checkout part
# we look for the right branch
# Branch name cannot have a ":"
# git ls-remote $REMOTE $BRANCH_NAME
####################################
file ( WRITE "${OTB_SOURCE_DIR}/sb_branch.txt" "${IMAGE_NAME}/${SB_MD5}")
message( "Checking out git for existence of archive")
set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact/")
set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}")
set( GIT "git" )
execute_process(
COMMAND ${GIT} "ls-remote" "${REMOTE}" "${BRANCH_NAME}"
OUTPUT_VARIABLE IS_SB_BUILD
)
if ( IS_SB_BUILD )
message( "Superbuild is already build for ${IMAGE_NAME} with sources as ${SB_MD5}")
return()
else()
message( "No build available, this job will build and push OTB_DEPENDS")
endif()
####################################
# Back to build
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET "OTB_DEPENDS"
RETURN_VALUE _build_rv
NUMBER_ERRORS _build_nb_err
CAPTURE_CMAKE_ERROR _build_error
)
if ( DEBUG )
message( "Status for build:" )
message("_build_rv=${_build_rv}")
message("_build_nb_err=${_build_nb_err}")
message("_build_error=${_build_error}")
endif()
if ( ( NOT ${_build_nb_err} EQUAL 0 ) OR ( ${_build_error} EQUAL -1 ))
ctest_submit()
message( FATAL_ERROR "An error occurs during ctest_build.")
endif()
ctest_submit()
########################################################################
########################################################################
# Git process
########################################################################
########################################################################
# WE PUSH ONLY IF BUILD SUCCEED
# The image used will be passed to this script.
# TODO verify that images does not have forbidden char in there name
# TODO right now we rely on ctest_build to know whether there has been an error
# in build, whereas SuperBuild does not necessarily return an error if something
# goes wrong
set ( SB_ARTIFACT_GIT "${CI_PROJ_DIR}/superbuild-artifact" )
# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are
# different because one is https and one is ssh. Both should be ssh.
set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git")
# We clone master to have a basic configuration, mainly a correct .gitattribute
# git clone $REMOTE --branch master --depth 1 superbuild-artifact
execute_process(
COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}"
"--branch" "master" "--depth" "1" "superbuild-artifact"
WORKING_DIRECTORY "${CI_PROJ_DIR}"
)
# setting up the repo
# StrictHostKeyChecking so we don't have to add the host as a known key
# -F /dev/null so the agent is not taking a default file ~/.ssh/..
execute_process(
COMMAND ${GIT} "config" "core.sshCommand"
"ssh -o StrictHostKeyChecking=no -F /dev/null"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE ssh_res
OUTPUT_VARIABLE ssh_out
ERROR_VARIABLE ssh_err
)
if ( DEBUG )
message( "Step 1: ssh")
message( "ssh_res = ${ssh_res}" )
message( "ssh_out = ${ssh_out}" )
message( "ssh_err = ${ssh_err}" )
endif()
execute_process(
COMMAND ${GIT} "config" "user.mail" "otbbot@orfeo-toolbox.org"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE mail_res
OUTPUT_VARIABLE mail_out
ERROR_VARIABLE mail_err
)
if ( DEBUG )
message( "Step 2: mail")
message( "mail_res = ${mail_res}" )
message( "mail_out = ${mail_out}" )
message( "mail_err = ${mail_err}" )
endif()
execute_process(
COMMAND ${GIT} "config" "user.name" "otbbot"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE name_res
OUTPUT_VARIABLE name_out
ERROR_VARIABLE name_err
)
if ( DEBUG )
message( "Step 3: name")
message( "name_res = ${name_res}" )
message( "name_out = ${name_out}" )
message( "name_err = ${name_err}" )
endif()
# create a branche
execute_process(
COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE co_res
OUTPUT_VARIABLE co_out
ERROR_VARIABLE co_err
)
if ( DEBUG )
message( "Step 4: check-o")
message( "co_res = ${co_res}" )
message( "co_out = ${co_out}" )
message( "co_err = ${co_err}" )
endif()
set ( SB_TAR_NAME "SuperBuild_Install.tar" )
# create the tar
# We need to create tar in its directory to avoid weird name in file
# "tar: Removing leading `../../' from member names"
# WARNING
# We are creating a tar containing xdk/.., so when extracting the archive in
# an other environment the output file will be xdk... Obvious isn't it?
# Well... Not for everyone...
# May be for easier maintainability the tar name should be the same as the
# file inside.
execute_process(
COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_TAR_NAME}"
-- "${CTEST_INSTALL_DIRECTORY}"
WORKING_DIRECTORY ${CI_ROOT_DIR}
)
# We need to copy the tar file, as it is on a different partition in the gitlab
# context
file ( COPY "${CI_ROOT_DIR}/${SB_TAR_NAME}" DESTINATION "${SB_ARTIFACT_GIT}")
# In a near futur it might be nice to clean up the mess we made...
if ( DEBUG )
if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
else()
message("Tar file does not exist")
endif()
endif()
# add the file
execute_process(
COMMAND ${GIT} "add" "${SB_TAR_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE add_res
OUTPUT_VARIABLE add_out
ERROR_VARIABLE add_err
)
if ( DEBUG )
message( "Step 5: add")
message( "add_res = ${add_res}" )
message( "add_out = ${add_out}" )
message( "add_err = ${add_err}" )
endif()
# commit
# We need the author because otherwise the mail is wrong
# In our case if toto is deploying a key in superbuild-artifact repo
# the the mail will be toto's
execute_process(
COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\""
"-m" "\"New Superbuild for ${SB_MD5} on ${IMAGE_NAME}\""
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE com_res
OUTPUT_VARIABLE com_out
ERROR_VARIABLE com_err
)
if ( DEBUG )
message( "Step 6: com")
message( "com_res = ${com_res}" )
message( "com_out = ${com_out}" )
message( "com_err = ${com_err}" )
endif()
# This part is just for debug
if ( DEBUG )
execute_process(
COMMAND ${GIT} "log" "-1"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE log_res
OUTPUT_VARIABLE log_out
ERROR_VARIABLE log_err
)
message( "Step 6bis: log")
message( "log_res = ${log_res}" )
message( "log_out = ${log_out}" )
message( "log_err = ${log_err}" )
endif()
# push
# we should be able to do a simple : git push origin $BRANCH_NAME
execute_process(
COMMAND ${GIT} "push" "${REPOSITORY_GIT_URL}" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE push_res
OUTPUT_VARIABLE push_out
ERROR_VARIABLE push_err
)
if ( DEBUG )
message( "Step 7: push")
message( "push_res = ${push_res}" )
message( "push_out = ${push_out}" )
message( "push_err = ${push_err}" )
endif()
\ No newline at end of file
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For know OTB_DEPENDS is build with default value
set ( cmake_configure_option
"CMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION}
CMAKE_INSTALL_PREFIX:PATH=${CTEST_INSTALL_DIRECTORY}")
set ( temporary_option
"OTB_USE_LIBKML=OFF")
set(concat_options
"${cmake_configure_option}
${temporary_option}
")
#Transform the previous string in list
string (REPLACE "\n" ";" sb_options ${concat_options})
foreach(item ${sb_options})
set( SB_CONFIGURE_OPTIONS "${SB_CONFIGURE_OPTIONS}-D${item};")
endforeach(item)
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment