Commit 67b7478c authored by Julien Michel's avatar Julien Michel

Merge branch 'develop' into patch_locS1

parents 10d49d4f 04207852
......@@ -25,7 +25,12 @@ before_script:
# Checkout the expected branch
- git checkout $CI_COMMIT_REF_NAME
after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH
stages:
- precheck
- prepare
- build
.general:
......@@ -35,13 +40,18 @@ stages:
- runner_system_failure
- stuck_or_timeout_failure
native-build:
fast-build:
extends: .general
only: [merge_requests]
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
only: [merge_requests, branches]
stage: precheck
image: $CI_REGISTRY/gpasero/otb/otb-install-ubuntu-native
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout $CI_COMMIT_REF_NAME
- python3 CI/check_twin_pipelines.py
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
debian-build:
extends: .general
......@@ -49,20 +59,59 @@ debian-build:
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-build:
extends: .general
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-shark:18.04
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-shark
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm
artifacts:
paths:
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
build:ubuntu-llvm:
ubuntu-llvm:
only: [merge_requests]
extends: .common-build
build:ubuntu-llvm-wip:
except: [merge_requests]
extends: .common-build
superbuild-prepare:
only: [merge_requests]
extends: .general
stage: prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
before_script:
# This override the previous before_script
- git checkout $CI_COMMIT_REF_NAME
# We are now doing the git-lfs install
# This is done after the checkout so we avoid downloading Data
# But we need it to upload the archive
- git-lfs install
- git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot"
- eval $(ssh-agent -s)
- ssh-add <(echo "$K8S_SECRET_SSH")
# This is for debug, we are checking the owner of the ssh key
- ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
after_script:
- echo "Nothing to do for after_script"
artifacts:
expire_in: 24 hrs
paths:
# This recovers logs from superbuild build
- build/*/*/*/*.log
- sb_branch.txt
superbuild-build:
only: [merge_requests]
extends: .general
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
dependencies:
- superbuild-prepare
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os.path
import urllib.request
import urllib.parse
import glob
import re
import unittest
import sys
import json
import time
trace = False
class Handler:
# project
# site
# stamp
# name
def __init__ (self):
self.build_dir = ""
self.configure_path = ""
self.url = "https://cdash.orfeo-toolbox.org"
self.project = "OTB"
self.site = ""
self.stamp = ""
self.name = ""
self.buildid = ""
def GetConfigureFile (self , build_dir = "" ):
if build_dir == "":
build_dir = self.build_dir
build_dir = os.path.normpath(build_dir)
test_path = os.path.join( build_dir , "Testing/")
list_folder = os.listdir(test_path)
if trace:
print(list_folder)
configure_xml = ""
for folder in list_folder:
if folder == "Temporary" or folder == "TAG":
pass
configure_xml = os.path.join( test_path , folder , "Configure.xml" )
if os.path.exists( configure_xml ):
break
if os.path.exists( configure_xml ):
if trace:
print ( configure_xml )
configure_file = open( configure_xml, "r" )
content = configure_file.read()
configure_file.close()
print( content )
self.configure_path = configure_xml
return self.configure_path
print("Could not find the Configure.xml produced by ctest")
return
def GetSite (self , build_dir="" ):
"""
Site is corresponding to the Name field in the xml.
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetSite function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
site_regex = re.compile( "\\bName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)")
site = site_regex.search( content )
if trace:
print (site_regex)
print(site)
if site:
if trace:
print("site value \n" , site.group(1))
self.site = site.group(1)
return self.site
print("Could not retreive site value")
return
return
def GetName (self , build_dir = ""):
"""
This function is looking for the name information in the build tree:
which is BuildName
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetName function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
name_regex = re.compile( "\\bBuildName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
name = name_regex.search( content )
if trace:
print (name_regex)
print( name)
if name:
if trace:
print("name value \n" , name.group(1))
self.name = name.group(1)
return self.name
print("Could not retreive name value")
return
def GetStamp (self , build_dir = "" ):
"""
This function is looking for the stamp information in the build tree
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetStamp function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
stamp_regex = re.compile( "\\bBuildStamp\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
stamp = stamp_regex.search( content )
if trace:
print( stamp_regex )
print( stamp )
if stamp:
if trace:
print("Stamp value \n" , stamp.group(1))
self.stamp = stamp.group(1)
return self.stamp
print("Could not retreive stamp value")
return
def GetBuildId (self, **kwargs):
"""
This function is returning the buildid. Dict can be passed with the
different informations
"""
site = self.site
stamp = self.stamp
name = self.name
project = self.project
for key , value in kwargs.items():
if key == "site":
site = value
if key == "stamp":
stamp = value
if key == "name":
name = value
if key == "project":
project = value
if ( site == "" or stamp == "" or name == "" or project == ""):
print( "Missing argument for buildid request \
site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
buildid_api = "/api/v1/getbuildid.php?"
buildid_params = urllib.parse.urlencode({'project': project, 'site': site, 'stamp': stamp , 'name': name})
full_url = self.url + buildid_api + buildid_params
if trace:
print("full_url: "+full_url)
nb_try = 6
build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
while nb_try:
response = urllib.request.urlopen(full_url).read().decode()
if trace:
print ( "response: " + response )
buildid = build_id_regex.search( response )
nb_try -= 1
if buildid or (nb_try == 0):
break
print("No build id, retry ...")
time.sleep(60)
if buildid:
self.buildid = buildid.group(1)
if trace:
print ( "build id is ", self.buildid)
return buildid.group(1)
else:
print("Error in recovering buildid")
return
def GetBuildUrl (self , buildid = "" ):
"""
This function is returning the build url. It can be called only when
everything is set
"""
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Missing argument to build url")
return
build_url = self.url
build_url +="/buildSummary.php?"
build_url += "buildid=" + buildid
return build_url
def GetBuildStatus(self, buildid = "" ):
"""
This function returns the status of a build id as a pair 'state' + 'errors'
"""
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Missing argument to build Status")
return
full_url = self.url + "/api/v1/buildSummary.php?buildid=" + buildid
response = urllib.request.urlopen(full_url).read().decode()
full_status = json.loads(response)
state = "success"
if full_status["configure"]["nerrors"] or full_status["build"]["nerrors"] \
or full_status["test"]["nerrors"] or full_status["test"]["nfailed"]:
state = 'failed'
errors = ""
if full_status["configure"]["nerrors"]:
errors = "Errors occur during configure"
elif full_status["build"]["nerrors"]:
errors = "Errors occur during build"
elif full_status["test"]["nerrors"] or full_status["test"]["nfailed"]:
errors = "Errors occur during tests"
return ( state , errors)
"""
TODO :
documentation, header, test if it is possible.
the script aims only at recovering the build url
"""
if __name__ == "__main__":
if ( len(sys.argv) < 5 ):
print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token")
sys.exit()
if trace:
print (sys.argv)
handler = Handler()
build_dir = os.path.join( sys.argv[3] , "build/")
# build_dir = sys.argv[3]
if trace:
print("build_dir is: " + build_dir)
handler.build_dir = build_dir
handler.GetSite()
handler.GetName()
handler.GetStamp()
if handler.GetBuildId() is None:
cdash_url = "https://cdash.orfeo-toolbox.org"
state = 'failed'
error = "Failed to get build id"
else:
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetBuildStatus()
if trace:
print ( "cdash_url is: " + cdash_url )
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url += sys.argv[2] + "/statuses/" + sys.argv[1]
params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\
'target_url' : cdash_url , 'description' : error })
gitlab_request = urllib.request.Request(gitlab_url)
gitlab_request.add_header('PRIVATE-TOKEN' , sys.argv[4] )
res = urllib.request.urlopen(gitlab_request, data=params.encode('ascii'))
if trace:
print ("gitlab_request.url: " + gitlab_request.full_url)
print ("gitlab_request.text: " + res.read().decode())
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import urllib.request
import urllib.parse
import json
import re
import time
"""
Send a request to Gitlab and return the answer
The request parameter is added after `project/:id/`
WARNING: when data is given, the request will be a POST
Otherwise, it is a GET
"""
def GitlabRequest(request, project=53, data=None, token=''):
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url+= str(project) + '/' + request
params = None
myHeader = {}
if not data is None:
params = urllib.parse.urlencode(data).encode('ascii')
if token:
myHeader = {'PRIVATE-TOKEN':token}
gitlab_request = urllib.request.Request(gitlab_url, data=params, headers=myHeader)
res = urllib.request.urlopen(gitlab_request)
return json.loads(res.read().decode())
"""
Check needed environment parameters
"""
def CheckEnvParameters(params):
for p in params:
if not p in os.environ.keys():
print("Missing environment variable '"+p+"'")
return False
return True
"""
Check for any duplicated twin pipeline and cancel it
"""
if __name__ == "__main__":
if not CheckEnvParameters(['CI_COMMIT_SHA']):
sys.exit(1)
env = os.environ
sha1 = env['CI_COMMIT_SHA']
# are we in a merge_request pipeline ?
if 'CI_MERGE_REQUEST_IID' in env.keys():
if not CheckEnvParameters(['K8S_SECRET_TWIN_PIPELINE','CI_PROJECT_ID','CI_PIPELINE_ID']):
sys.exit(1)
mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_TWIN_PIPELINE'])
wip_regex = re.compile("^[Ww][Ii][Pp]:")
# is it a "WIP" merge request ?
if wip_regex.search(mrInfo["title"]):
# Yes: cancel the current pipeline
print("Cancel current pipeline "+env['CI_PIPELINE_ID'])
GitlabRequest('pipelines/'+env['CI_PIPELINE_ID']+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
time.sleep(180)
print("Error: this pipeline should have been canceled")
sys.exit(1)
else:
# No: cancel any previous "normal" pipeline on the same SHA1
jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
for item in jres:
if item["id"] < int(env['CI_PIPELINE_ID']) and item["status"] == "running":
print("Cancel pipeline "+str(item["id"]))
jres2 = GitlabRequest('pipelines/'+str(item["id"])+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
......@@ -51,12 +51,12 @@ OTB_USE_QWT:BOOL=ON
OTB_USE_SHARK:BOOL=ON
OTB_USE_SIFTFAST:BOOL=ON
OTB_USE_SPTW:BOOL=ON
OTB_USE_SSE_FLAGS:BOOL=ON")
OTB_USE_SSE_FLAGS:BOOL=ON
OTB_MPIEXEC_OPT:STRING=--allow-run-as-root")
set (otb_wrap_option
"OTB_WRAP_JAVA:BOOL=ON
OTB_WRAP_PYTHON:BOOL=ON
OTB_WRAP_PYTHON3:BOOL=ON")
OTB_WRAP_PYTHON:BOOL=ON")
set (otb_data_option
"OTB_DATA_USE_LARGEINPUT:BOOL=OFF
......
......@@ -26,13 +26,22 @@ set (ENV{LANG} "C") # Only ascii output
set (CTEST_BUILD_CONFIGURATION "Release")
set (CTEST_CMAKE_GENERATOR "Ninja")
# detect short sha
if(NOT DEFINED ENV{CI_COMMIT_SHORT_SHA})
execute_process(COMMAND git log -1 --pretty=format:%h
WORKING_DIRECTORY ${OTB_SOURCE_DIR}
OUTPUT_VARIABLE ci_short_sha)
else()
set(ci_short_sha "$ENV{CI_COMMIT_SHORT_SHA}")
endif()
# Find the build name and CI profile
set(ci_profile wip)
set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}")
set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
set (CTEST_BUILD_NAME "$ENV{CI_COMMIT_SHORT_SHA}")
set (CTEST_BUILD_NAME ${ci_short_sha})
if(ci_mr_source AND ci_mr_target AND ci_mr_iid)
set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid})")
set(ci_profile mr)
......@@ -45,12 +54,37 @@ elseif(ci_ref_name)
endif()
endif()
# set pipelines to enable documentation
set(ci_cookbook_profiles mr develop release)
set(ci_doxygen_profiles mr develop release)
list(FIND ci_cookbook_profiles ${ci_profile} ci_do_cookbook)
list(FIND ci_doxygen_profiles ${ci_profile} ci_do_doxygen)
# Detect site
if(NOT DEFINED IMAGE_NAME)
if(DEFINED ENV{IMAGE_NAME})
set(IMAGE_NAME $ENV{IMAGE_NAME})
endif()
endif()
set (CTEST_SITE "${IMAGE_NAME}")
# Detect "skip testing"
if(DEFINED ENV{CI_SKIP_TESTING})
set(ci_skip_testing 1)
endif()
# Directory variable
set (CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}")
set (CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/")
set (CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install/")
if(BUILD_DIR)
set (CTEST_BINARY_DIRECTORY "${BUILD_DIR}")
else()
set (CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/")
endif()
if(INSTALL_DIR)
set (CTEST_INSTALL_DIRECTORY "${INSTALL_DIR}")
else()
set (CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/install/")
endif()
set (PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}")
# Ctest command value
......@@ -62,6 +96,7 @@ set (OTB_LARGEINPUT_ROOT "") # todo
message(STATUS "CI profile : ${ci_profile}")
#The following file set the CONFIGURE_OPTIONS variable
set (ENABLE_DOXYGEN OFF)
set (CONFIGURE_OPTIONS "")
include ( "${CMAKE_CURRENT_LIST_DIR}/configure_option.cmake" )
......@@ -86,6 +121,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
)
if ( NOT _configure_rv EQUAL 0 )
# stop processing here
ctest_submit()
message( FATAL_ERROR "An error occurs during ctest_configure.")
endif()
......@@ -96,19 +132,30 @@ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
)
if ( NOT _build_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_build.")
endif()
# Uncomment when ready for test
# ctest_test(PARALLEL_LEVEL 8
# RETURN_VALUE _test_rv
# CAPTURE_CMAKE_ERROR _test_error
# )
if(ci_skip_testing)
message(STATUS "Skip testing")
set(_test_rv 0)
else()
ctest_test(PARALLEL_LEVEL 8
RETURN_VALUE _test_rv
CAPTURE_CMAKE_ERROR _test_error
)
endif()
# if ( NOT _test_rv EQUAL 0 )
# ctest_submit()
# message( SEND_ERROR "An error occurs during ctest_test.")
# endif()
if ( NOT _test_rv EQUAL 0 )
message( SEND_ERROR "An error occurs during ctest_test.")
endif()
ctest_submit()
if(ENABLE_DOXYGEN)
# compile doxygen
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET Documentation
RETURN_VALUE _doxy_rv
CAPTURE_CMAKE_ERROR _doxy_error
)
endif()
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is a prototype for the future CI, it may evolve rapidly in a near future
set (ENV{LANG} "C") # Only ascii output
get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
get_filename_component(CI_PROJ_DIR ${OTB_SOURCE_DIR} DIRECTORY)
get_filename_component(CI_ROOT_DIR ${CI_PROJ_DIR} DIRECTORY)
###########################################################################
###########################################################################
# Download xkd
###########################################################################
###########################################################################
# How to get md5sum:
# * concatenate all source files in one
# * add configure result from previous job ${OTB_SOURCE_DIR}/build/CMakeCache.txt
###########################
file(READ "${OTB_SOURCE_DIR}/sb_branch.txt" BRANCH_NAME)
####################################
# git clone $REPOSITORY_URL --branch $BRANCH_NAME --depth 1 superbuild-artifact
###############################################################################
set ( REMOTE "https://gitlab.orfeo-toolbox.org/gbonnefille/superbuild-artifact.git")
# set ( BRANCH_NAME "${IMAGE_NAME}/${SB_MD5}")
set( GIT "git" )