Commit b507b210 authored by Guillaume Pasero's avatar Guillaume Pasero
Browse files

MRG: Synchronize with branch develop

Conflicts:
	.gitlab-ci.yml
parents 90077dd9 1bb82bdd
......@@ -53,6 +53,7 @@ fast-build:
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
......@@ -67,6 +68,7 @@ fast-build:
- /^release-[0-9]+\.[0-9]+$/
stage: build
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
......@@ -153,3 +155,66 @@ macos-xdk-build:
- ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies:
- macos-xdk-prepare
## Windows
.windows-prepare:
extends: .common-prepare
before_script:
# This override the previous before_script
- set GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q %CI_COMMIT_SHA%
- set GIT_LFS_SKIP_SMUDGE=0
.windows-build:
extends: .common-build
before_script:
- git lfs fetch origin %CI_COMMIT_SHA%
- git checkout -f -q %CI_COMMIT_SHA%
# - Win10
windows-10-prepare:
extends: .windows-prepare
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 xdk 10
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-10-build:
extends: .windows-build
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 otb 10
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-10-prepare
# - Win8.1
windows-8-prepare:
extends: .windows-prepare
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 xdk 8.1
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-8-build:
extends: .windows-build
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 otb 8.1
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-8-prepare
@echo off
:: check input arguments
if %1.==. (
echo "No arch"
call :Help
goto :eof
)
if %2.==. (
echo "No project"
call :Help
goto :eof
)
if /I "%1"=="help" (
call :Help
goto :eof
)
if /I "%1"=="/help" (
call :Help
goto :eof
)
if /I "%1"=="-help" (
call :Help
goto :eof
)
set ARCH=%1
set PROJECT=%2
if %3.==. (
set SHORT_TARGET=10
) else (
set SHORT_TARGET=%3
)
if %4.==. (
set VCVER=14.0
) else (
set VCVER=%4
)
set TARGET=%SHORT_TARGET%
if "%TARGET%"=="10" (
set TARGET=10.0.17763.0
)
:: Setup home dir (so that ssh configuration works fine)
:: if "%USERNAME%"=="otbbot" (
set HOMEDRIVE=C:
set HOMEPATH=\Users\otbbot
::)
echo Home dir: %HOMEDRIVE%%HOMEPATH%
:: Setup Python
set PATH=C:\tools\Python35-%ARCH%;%PATH%
set PATH=C:\tools\Python35-%ARCH%\Scripts;%PATH%
:: Setup GL dlls
set PATH=%PATH%;C:\tools\GL\%ARCH%\bin
:: Setup compiler
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" %ARCH% %TARGET% -vcvars_ver=%VCVER%
:: Setup Clcache
set CLCACHE_DIR=C:\clcache\%PROJECT%-%ARCH%-%TARGET%-%VCVER%
set CLCACHE_HARDLINK=1
:: set CLCACHE_SERVER=1
set CLCACHE_CL=
for /F "delims=" %%a in ('where cl.exe') do @if defined CLCACHE_CL (break ) else (set CLCACHE_CL=%%a)
echo CL path: "%CLCACHE_CL%"
:: install clcache.exe as cl.exe
copy C:\tools\Python35-%ARCH%\Scripts\clcache.exe C:\clcache\cl.exe
set PATH=C:\clcache;%PATH%
:: we need to change cache max size: clcache -M <size-in-bytes>
if "%PROJECT%"=="xdk" (
call "clcache.exe" -M 3000000000
)
if "%PROJECT%"=="otb" (
call "clcache.exe" -M 2000000000
)
set IMAGE_NAME=windows-%SHORT_TARGET%-%ARCH%-vc%VCVER%
echo Generated IMAGE_NAME: %IMAGE_NAME%
:: setup path to perl, but add it last ... (there is a libstdc++.dll in that folder...)
set PATH=%PATH%;C:\tools\perl\perl\bin
goto :eof
:Help
setlocal
echo "Usage: dev_env.bat <compiler_arch> <project> [<target-os> [<vc_version>]]"
echo " <compiler_arch> : 'x86' | 'x64'"
echo " <project> : 'xdk' | 'otb'"
echo " <target-os> : '8.1' | '10' (default)"
echo " <vc_version> :"
echo " '14.20' (i.e. VS 2019)"
echo " '14.16' (i.e. VS 2017)"
echo " '14.0' (i.e. VS 2015) (default)"
endlocal
......@@ -29,9 +29,20 @@ get_filename_component( OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY )
set ( DEBUG "1" )
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j1" )
if(WIN32)
set ( CTEST_CMAKE_GENERATOR "NMake Makefiles JOM" )
set ( CTEST_BUILD_FLAGS "/S" )
else()
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j1")
endif()
set ( CTEST_BUILD_NAME "Packages" )
# Detect site
if(NOT DEFINED IMAGE_NAME)
if(DEFINED ENV{IMAGE_NAME})
set(IMAGE_NAME $ENV{IMAGE_NAME})
endif()
endif()
set ( CTEST_SITE "${IMAGE_NAME}" )
# Find the build name and CI profile
......@@ -62,7 +73,7 @@ set ( CONFIGURE_OPTIONS
find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
# Sources are already checked out : do nothing for update
set(CTEST_GIT_UPDATE_CUSTOM echo No update)
set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "No update")
ctest_start( Experimental TRACK CI_Package )
......
......@@ -32,17 +32,31 @@ get_xdk()
set( INSTALL_DIR "${XDK_PATH}" )
# FIX ME this part might platform dependent
set( GDAL_DATA "${XDK_PATH}/share/gdal" )
set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
set( PROJ_LIB "${XDK_PATH}/share" )
set( CTEST_ENVIRONMENT
"PATH=${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}
if(WIN32)
file(TO_NATIVE_PATH "${XDK_PATH}" XDK_PATH_NATIVE)
file(TO_NATIVE_PATH "${CTEST_BINARY_DIRECTORY}/bin" OTB_BUILD_BIN_DIR_NATIVE)
set(ENV{PATH} "$ENV{PATH};${OTB_BUILD_BIN_DIR_NATIVE}" )
set(ENV{PATH} "${XDK_PATH_NATIVE}\\bin;$ENV{PATH}" )
set(ENV{PATH} "$ENV{PATH};${XDK_PATH_NATIVE}\\lib" )
set(ENV{GDAL_DATA} "${XDK_PATH_NATIVE}\\data" )
set(ENV{GEOTIFF_CSV} "${XDK_PATH_NATIVE}\\share\\epsg_csv" )
set(ENV{PROJ_LIB} "${XDK_PATH_NATIVE}\\share" )
# needed to load Qt plugins for testing, not for binary packages where we use a qt.conf file
set(ENV{QT_PLUGIN_PATH} "${XDK_PATH_NATIVE}\\plugins")
set( CTEST_ENVIRONMENT
"PATH=$ENV{PATH}
GDAL_DATA=$ENV{GDAL_DATA}
GEOTIFF_CSV=$ENV{GEOTIFF_CSV}
PROJ_LIB=$ENV{PROJ_LIB}
")
# It seems that we do not need that
# GDAL_DATA= GDAL_DATA
# GEOTIFF_CSV= GEOTIFF_CSV
# PROJ_LIB= PROJ_LIB
else()
set(ENV{PATH} "${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}" )
set( GDAL_DATA "${XDK_PATH}/share/gdal" )
set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
set( PROJ_LIB "${XDK_PATH}/share" )
set( CTEST_ENVIRONMENT
"PATH=$ENV{PATH}
")
endif()
include( "${CMAKE_CURRENT_LIST_DIR}/main_ci.cmake" )
......@@ -27,10 +27,22 @@ get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" )
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
if(WIN32)
set ( CTEST_CMAKE_GENERATOR "NMake Makefiles JOM" )
set ( CTEST_BUILD_FLAGS "/S" )
else()
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j16")
endif()
set ( PROJECT_SOURCE_DIR "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_SOURCE_DIRECTORY "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
# Detect site
if(NOT DEFINED IMAGE_NAME)
if(DEFINED ENV{IMAGE_NAME})
set(IMAGE_NAME $ENV{IMAGE_NAME})
endif()
endif()
set ( CTEST_SITE "${IMAGE_NAME}" )
......@@ -48,9 +60,16 @@ set (CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/xdk/")
# HACK
# This is needed because when using return() function ctest is trying
# to run the CTEST_COMMAND. And we need it to not produce an error
set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
##set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
set (CMAKE_COMMAND "cmake")
if(WIN32)
file(TO_NATIVE_PATH "${CTEST_INSTALL_DIRECTORY}" XDK_INSTALL_DIR_NATIVE)
set(ENV{PATH} "${XDK_INSTALL_DIR_NATIVE}\\bin;$ENV{PATH}" )
set(ENV{PATH} "$ENV{PATH};${XDK_INSTALL_DIR_NATIVE}\\lib" )
# set(ENV{CMAKE_PREFIX_PATH} "${XDK_INSTALL_DIR}" )
endif()
########################################################################
########################################################################
# Build process
......@@ -64,14 +83,12 @@ find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
set( GIT "${CTEST_GIT_COMMAND}" )
# Sources are already checked out : do nothing for update
set(CTEST_GIT_UPDATE_CUSTOM echo No update)
set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "No update")
ctest_start (Experimental TRACK CI_Prepare)
ctest_update( SOURCE "${OTB_SOURCE_DIR}" )
set(CTEST_BUILD_FLAGS "-j16")
set ( SB_CONFIGURE_OPTIONS "")
include( "${CMAKE_CURRENT_LIST_DIR}/sb_configure_options.cmake" )
......@@ -84,8 +101,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
if ( NOT _configure_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_configure. Dependencies might be buggy.")
return()
message( FATAL_ERROR "An error occurs during ctest_configure. Dependencies might be buggy.")
endif()
########################################################################
......@@ -130,160 +146,167 @@ execute_process(
)
if ( IS_SB_BUILD )
message( "Superbuild is already build for ${IMAGE_NAME} with sources as ${SB_MD5}")
return()
else()
message( "No build available, this job will build and push OTB_DEPENDS")
endif()
####################################
# Back to build
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET "OTB_DEPENDS"
RETURN_VALUE _build_rv
NUMBER_ERRORS _build_nb_err
CAPTURE_CMAKE_ERROR _build_error
)
if ( DEBUG )
message( "Status for build:" )
message("_build_rv=${_build_rv}")
message("_build_nb_err=${_build_nb_err}")
message("_build_error=${_build_error}")
endif()
if ( ( NOT ${_build_nb_err} EQUAL 0 ) OR ( ${_build_error} EQUAL -1 ))
message( FATAL_ERROR "An error occurs during ctest_build.")
endif()
ctest_submit()
########################################################################
########################################################################
# Git process
########################################################################
########################################################################
# WE PUSH ONLY IF BUILD SUCCEED
# The image used will be passed to this script.
# TODO verify that images does not have forbidden char in there name
# TODO right now we rely on ctest_build to know whether there has been an error
# in build, whereas SuperBuild does not necessarily return an error if something
# goes wrong
# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are
# different because one is https and one is ssh. Both should be ssh.
set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git")
# We clone master to have a basic configuration, mainly a correct .gitattribute
# git clone $REMOTE --branch master --depth 1 superbuild-artifact
execute_process(
COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}"
"--branch" "master" "--depth" "1" "superbuild-artifact"
WORKING_DIRECTORY "${OTB_SOURCE_DIR}"
)
set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" )
# create a branche
execute_process(
COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE co_res
OUTPUT_VARIABLE co_out
ERROR_VARIABLE co_err
)
if ( DEBUG )
message( "Step 4: check-o")
message( "co_res = ${co_res}" )
message( "co_out = ${co_out}" )
message( "co_err = ${co_err}" )
endif()
set ( SB_TAR_NAME "SuperBuild_Install.tar" )
# Creating the tar
# May be for easier maintainability the tar name should be the same as the
# file inside.
execute_process(
COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}"
-- "${CTEST_INSTALL_DIRECTORY}"
WORKING_DIRECTORY ${OTB_SOURCE_DIR}
)
####################################
# Back to build
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET "OTB_DEPENDS"
RETURN_VALUE _build_rv
NUMBER_ERRORS _build_nb_err
CAPTURE_CMAKE_ERROR _build_error
)
if ( DEBUG )
message( "Status for build:" )
message("_build_rv=${_build_rv}")
message("_build_nb_err=${_build_nb_err}")
message("_build_error=${_build_error}")
endif()
if ( ( NOT ${_build_nb_err} EQUAL 0 ) OR ( ${_build_error} EQUAL -1 ))
ctest_submit()
message( FATAL_ERROR "An error occurs during ctest_build.")
endif()
ctest_submit()
########################################################################
########################################################################
# Git process
########################################################################
########################################################################
# WE PUSH ONLY IF BUILD SUCCEED
# The image used will be passed to this script.
# TODO verify that images does not have forbidden char in there name
# TODO right now we rely on ctest_build to know whether there has been an error
# in build, whereas SuperBuild does not necessarily return an error if something
# goes wrong
set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" )
# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are
# different because one is https and one is ssh. Both should be ssh.
set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git")
# We clone master to have a basic configuration, mainly a correct .gitattribute
# git clone $REMOTE --branch master --depth 1 superbuild-artifact
execute_process(
COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}"
"--branch" "master" "--depth" "1" "superbuild-artifact"
WORKING_DIRECTORY "${OTB_SOURCE_DIR}"
)
if ( DEBUG )
if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
else()
message("Tar file does not exist")
# create a branche
execute_process(
COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE co_res
OUTPUT_VARIABLE co_out
ERROR_VARIABLE co_err
)
if ( DEBUG )
message( "Step 4: check-o")
message( "co_res = ${co_res}" )
message( "co_out = ${co_out}" )
message( "co_err = ${co_err}" )
endif()
endif()
set ( SB_TAR_NAME "SuperBuild_Install.tar" )
# create the tar
# We need to create tar in its directory to avoid weird name in file
# "tar: Removing leading `../../' from member names"
# WARNING
# We are creating a tar containing xdk/.., so when extracting the archive in
# an other environment the output file will be xdk... Obvious isn't it?
# Well... Not for everyone...
# May be for easier maintainability the tar name should be the same as the
# file inside.
execute_process(
COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "superbuild-artifact/${SB_TAR_NAME}"
-- "${CTEST_INSTALL_DIRECTORY}"
WORKING_DIRECTORY ${OTB_SOURCE_DIR}
)
# add the file
execute_process(
COMMAND ${GIT} "add" "${SB_TAR_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE add_res
OUTPUT_VARIABLE add_out
ERROR_VARIABLE add_err
)
# In a near futur it might be nice to clean up the mess we made...
if ( DEBUG )
message( "Step 5: add")
message( "add_res = ${add_res}" )
message( "add_out = ${add_out}" )
message( "add_err = ${add_err}" )
endif()
if ( DEBUG )
if (EXISTS "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
message("Tar file exists in superbuild_artefact at: ${SB_ARTIFACT_GIT}/${SB_TAR_NAME}")
else()
message("Tar file does not exist")
endif()
endif()
# add the file
execute_process(
COMMAND ${GIT} "add" "${SB_TAR_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE add_res
OUTPUT_VARIABLE add_out
ERROR_VARIABLE add_err
)
if ( DEBUG )
message( "Step 5: add")
message( "add_res = ${add_res}" )
message( "add_out = ${add_out}" )
message( "add_err = ${add_err}" )
endif()
# commit
# We need the author because otherwise the mail is wrong
# In our case if toto is deploying a key in superbuild-artifact repo
# the the mail will be toto's
execute_process(
COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\""
"-m" "\"New Superbuild for ${SB_MD5} on ${IMAGE_NAME}\""
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE com_res
OUTPUT_VARIABLE com_out
ERROR_VARIABLE com_err
)
# commit
# We need the author because otherwise the mail is wrong
# In our case if toto is deploying a key in superbuild-artifact repo
# the the mail will be toto's
execute_process(
COMMAND ${GIT} "commit" "--author=\"otbbot <otbbot@orfeo-toolbox.org>\""
"-m" "\"New Superbuild for ${SB_MD5} on ${IMAGE_NAME}\""
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE com_res
OUTPUT_VARIABLE com_out
ERROR_VARIABLE com_err
)
if ( DEBUG )
message( "Step 6: com")
message( "com_res = ${com_res}" )
message( "com_out = ${com_out}" )
message( "com_err = ${com_err}" )
endif()
if ( DEBUG )
message( "Step 6: com")
message( "com_res = ${com_res}" )
message( "com_out = ${com_out}" )
message( "com_err = ${com_err}" )
endif()
# This part is just for debug
if ( DEBUG )
execute_process(
COMMAND ${GIT} "log" "-1"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE log_res
OUTPUT_VARIABLE log_out
ERROR_VARIABLE log_err
)
message( "Step 6bis: log")
message( "log_res = ${log_res}" )
message( "log_out = ${log_out}" )
message( "log_err = ${log_err}" )
endif()
# This part is just for debug
if ( DEBUG )
# push
# we should be able to do a simple : git push origin $BRANCH_NAME
execute_process(
COMMAND ${GIT} "log" "-1"
COMMAND ${GIT} "push" "${REPOSITORY_GIT_URL}" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE log_res
OUTPUT_VARIABLE log_out
ERROR_VARIABLE log_err
RESULT_VARIABLE push_res
OUTPUT_VARIABLE push_out
ERROR_VARIABLE push_err
)
message( "Step 6bis: log")
message( "log_res = ${log_res}" )
message( "log_out = ${log_out}" )
message( "log_err = ${log_err}" )
endif()
# push
# we should be able to do a simple : git push origin $BRANCH_NAME
execute_process(
COMMAND ${GIT} "push" "${REPOSITORY_GIT_URL}" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE push_res
OUTPUT_VARIABLE push_out