Skip to content

GitLab

Commit 61f55ac1 authored by Guillaume Pasero's avatar Guillaume Pasero
Browse files

Merge branch 'ci_alpha_fixes' into 'develop' 

CI Fixes episode N

See merge request !536
parents 4facfd77 167c91e6
Pipeline #2035 passed with stages
in 6 minutes and 44 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 61f55ac1
......@@ -34,26 +34,11 @@ stages:
.common:
extends: .general
before_script:
- git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 CI/cdash_handler.py
fast-build:
extends: .common
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
- python3 -u CI/cdash_handler.py
.common-build:
extends: .common
......@@ -74,12 +59,6 @@ fast-build:
- build_packages/OTB-*.run
- build_packages/OTB-*.zip
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-prepare:
extends: .general
only:
......@@ -88,7 +67,6 @@ debian-build:
- /^release-[0-9]+\.[0-9]+$/
stage: prepare
before_script:
- git lfs install --skip-repo
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
......@@ -99,6 +77,27 @@ debian-build:
- sb_branch.txt # Needed to checkout correct branch in build step
- build/*/*/*/*.log # Superbuild log
#-------------------------- precheck job ---------------------------------------
fast-build:
extends: .common
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
#------------------------- prepare & build jobs --------------------------------
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
## Ubuntu superbuild
ubuntu-xdk-prepare:
extends: .common-prepare
......@@ -111,7 +110,7 @@ ubuntu-xdk-build:
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
dependencies:
- ubuntu-xdk-prepare
......@@ -127,20 +126,15 @@ centos-xdk-build:
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies:
- centos-xdk-prepare
## MacOS superbuild
macos-xdk-prepare:
extends: .common-prepare
tags:
- macos
extends: .common-prepare
before_script:
# No need to install lfs as this machine is persistent
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
......@@ -150,7 +144,7 @@ macos-xdk-build:
extends: .common-build
script:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies:
- macos-xdk-prepare
......@@ -217,9 +211,7 @@ windows-8-build:
dependencies:
- windows-8-prepare
# Deploy job
#---------------------------- Deploy job ---------------------------------------
deploy:
tags:
- deploy
......@@ -227,13 +219,11 @@ deploy:
stage: deploy
extends: .general
only:
- develop
- /^release-[0-9]+\.[0-9]+$/
- develop@orfeotoolbox/otb
- /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
before_script:
- git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout -f -q $CI_COMMIT_SHA
script:
- ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
......@@ -242,4 +232,16 @@ deploy:
- centos-xdk-build
- macos-xdk-build
- windows-8-build
- windows-10-build
\ No newline at end of file
- windows-10-build
release-container:
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
refs:
- tags@orfeotoolbox/otb
variables:
- $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
script:
- "curl --request POST --form token=$K8S_SECRET_RELEASE --form ref=master --form variables[OTB_TAG]=$CI_COMMIT_TAG https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline"
CI/cdash_handler.py
View file @ 61f55ac1
......@@ -28,6 +28,7 @@ import unittest
import sys
import json
import time
import xml.etree.ElementTree as ET
trace = False
......@@ -42,11 +43,10 @@ def CheckEnvParameters(params):
return False
return True
"""
Handler class to retrieve build informations
"""
class Handler:
# project
# site
# stamp
# name
def __init__ (self):
self.build_dir = ""
self.configure_path = ""
......@@ -75,91 +75,39 @@ class Handler:
if os.path.exists( configure_xml ):
if trace:
print ( configure_xml )
configure_file = open( configure_xml, "r" )
content = configure_file.read()
configure_file.close()
print( content )
self.configure_path = configure_xml
return self.configure_path
print("Could not find the Configure.xml produced by ctest")
return
def GetSite (self , build_dir="" ):
"""
Site is corresponding to the Name field in the xml.
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetSite function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
site_regex = re.compile( "\\bName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)")
site = site_regex.search( content )
if trace:
print (site_regex)
print(site)
if site:
if trace:
print("site value \n" , site.group(1))
self.site = site.group(1)
return self.site
print("Could not retreive site value")
return
return
def GetName (self , build_dir = ""):
"""
This function is looking for the name information in the build tree:
which is BuildName
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetName function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
name_regex = re.compile( "\\bBuildName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
name = name_regex.search( content )
if trace:
print (name_regex)
print( name)
if name:
if trace:
print("name value \n" , name.group(1))
self.name = name.group(1)
return self.name
print("Could not retreive name value")
return
sys.exit(1)
def GetStamp (self , build_dir = "" ):
def ParseConfigureFile(self):
"""
This function is looking for the stamp information in the build tree
Parse the configuration file to get Name, Site and BuildStamp
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetStamp function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
stamp_regex = re.compile( "\\bBuildStamp\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
stamp = stamp_regex.search( content )
# strip the Log section as it can mess up the XML parser
startLog=content.find('<Log>')
endLog=content.rfind('</Log>')
if startLog > 0 and endLog > startLog:
content = content[:(startLog+5)]+content[endLog:]
# parse XML
root = ET.fromstring(content)
if trace:
print( stamp_regex )
print( stamp )
if stamp:
if trace:
print("Stamp value \n" , stamp.group(1))
self.stamp = stamp.group(1)
return self.stamp
print("Could not retreive stamp value")
return
print( root.attrib )
if not 'Name' in root.keys():
print("Can't find site name in Configure.XML")
sys.exit(1)
if not 'BuildName' in root.keys():
print("Can't find build name in Configure.XML")
sys.exit(1)
if not 'BuildStamp' in root.keys():
print("Can't find build stamp in Configure.XML")
sys.exit(1)
self.site = root.get('Name')
self.name = root.get('BuildName')
self.stamp = root.get('BuildStamp')
def GetBuildId (self, **kwargs):
"""
......@@ -180,15 +128,17 @@ class Handler:
if key == "project":
project = value
if ( site == "" or stamp == "" or name == "" or project == ""):
print( "Missing argument for buildid request \
site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
print( "Missing argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
sys.exit(1)
elif trace:
print( "Argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
buildid_api = "/api/v1/getbuildid.php?"
buildid_params = urllib.parse.urlencode({'project': project, 'site': site, 'stamp': stamp , 'name': name})
full_url = self.url + buildid_api + buildid_params
if trace:
print("full_url: "+full_url)
nb_try = 6
max_retry = 11
nb_try = max_retry
build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
while nb_try:
response = urllib.request.urlopen(full_url).read().decode()
......@@ -198,8 +148,8 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
nb_try -= 1
if buildid or (nb_try == 0):
break
print("No build id, retry ...")
time.sleep(60)
print("No build id, retry "+str(max_retry-nb_try)+"/"+str(max_retry)+" ...")
time.sleep(30)
if buildid:
self.buildid = buildid.group(1)
if trace:
......@@ -207,7 +157,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return buildid.group(1)
else:
print("Error in recovering buildid")
return
sys.exit(1)
def GetBuildUrl (self , buildid = "" ):
"""
......@@ -290,11 +240,11 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
cdash_handler.py commit_sha1 project_id project_directory token ref_name
"""
if __name__ == "__main__":
if trace:
print(sys.argv)
if ( len(sys.argv) < 6 and len(sys.argv) > 1 ):
print("Usage : "+sys.argv[0]+" commit_sha1 project_id project_directory token ref_name")
sys.exit(1)
if trace:
print (sys.argv)
if ( len(sys.argv) >= 6):
sha1 = sys.argv[1]
proj = sys.argv[2]
......@@ -302,24 +252,26 @@ if __name__ == "__main__":
token = sys.argv[4]
refn = sys.argv[5]
else:
if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'K8S_SECRET_API_TOKEN', 'CI_COMMIT_REF_NAME']):
if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'CI_COMMIT_REF_NAME']):
sys.exit(1)
sha1 = os.environ['CI_COMMIT_SHA']
proj = os.environ['CI_PROJECT_ID']
pdir = os.environ['CI_PROJECT_DIR']
token = os.environ['K8S_SECRET_API_TOKEN']
if 'CI_MERGE_REQUEST_REF_PATH' in os.environ.keys():
refn = os.environ['CI_MERGE_REQUEST_REF_PATH']
else:
refn = os.environ['CI_COMMIT_REF_NAME']
if CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
token = os.environ['K8S_SECRET_API_TOKEN']
else:
token = None
handler = Handler()
build_dir = os.path.join( pdir , "build/")
if trace:
print("build_dir is: " + build_dir)
handler.build_dir = build_dir
handler.GetSite()
handler.GetName()
handler.GetStamp()
handler.GetConfigureFile()
handler.ParseConfigureFile()
if handler.GetBuildId() is None:
cdash_url = "https://cdash.orfeo-toolbox.org"
state = 'failed'
......@@ -327,8 +279,9 @@ if __name__ == "__main__":
else:
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetLogStatus( os.path.join( pdir , "log") )
if trace:
print ( "cdash_url is: " + cdash_url )
print("CDash build URL : "+cdash_url)
if token is None:
sys.exit(0)
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url += proj + "/statuses/" + sha1
params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\
......
CI/check_twin_pipelines.py
View file @ 61f55ac1
......@@ -24,6 +24,7 @@ import urllib.parse
import json
import re
import time
import sys
"""
Send a request to Gitlab and return the answer
......@@ -65,9 +66,10 @@ if __name__ == "__main__":
# are we in a merge_request pipeline ?
if 'CI_MERGE_REQUEST_IID' in env.keys():
if not CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
print("Make sure you have set a valid acces token for Gitlab API")
print("The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'")
sys.exit(1)
print("WARNING: Make sure you have set a valid acces token for Gitlab API." \
+ "The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'" \
+ "Without this token, some feature of the CI platform will be disabled.")
sys.exit(0)
if not CheckEnvParameters(['CI_PROJECT_ID','CI_PIPELINE_ID']):
sys.exit(1)
mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_API_TOKEN'])
......
Packaging/detect_using_file_command.cmake
View file @ 61f55ac1
......@@ -115,10 +115,10 @@ function(detect_using_file_command input_file result_type result_dir)
# Patch file command returning shared-oject for executable on Linux when -PIE is used.
if( LINUX )
get_filename_component( input_file_DIR ${input_file} DIRECTORY )
get_filename_component( input_file_dIR ${input_file_DIR} NAME )
string( TOLOWER "${input_file_DIR}" input_file_DIR )
if( ${input_file_DIR} MATCHES "bin" )
message( WARNING "${input_file} detected as shared-object." )
get_filename_component( dir_name ${input_file_DIR} NAME )
string( TOLOWER "${dir_name}" dir_name_l )
if( "${dir_name_l}" STREQUAL "bin" )
message( STATUS "${input_file} detected as shared-object, but processed as executable" )
set(detected_type PROGRAMS)
set(detected_dir bin)
endif()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment