Commit 453a86c2 authored by Antoine Regimbeau's avatar Antoine Regimbeau
Browse files

Merge branch 'develop' into qgis_parameter_status_2

parents 1afe1bc1 2e4c87da
......@@ -22,6 +22,8 @@ stages:
- precheck
- prepare
- build
- report
- deploy
.general:
retry:
......@@ -30,55 +32,49 @@ stages:
- runner_system_failure
- stuck_or_timeout_failure
fast-build:
.common:
extends: .general
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
after_script:
- python3 CI/cdash_handler.py
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
.common-build:
extends: .general
only: [merge_requests]
stage: build
before_script:
- git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 CI/cdash_handler.py
# artifacts:
# when: on_failure
# expire_in: 24 hrs
# paths:
# - build/*/*.log #CMake log
# - log/*.txt # Others
- python3 -u CI/cdash_handler.py
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-build:
extends: .common
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: build
artifacts:
when: always
expire_in: 24 hrs
paths:
- build/*/*.log #CMake log
- log/*.txt # Others
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
- build_packages/OTB-*.run
- build_packages/OTB-*.zip
- build/compile_commands.json
- build/ctest_report.xml
- build/cppcheck_report.xml
- build/coverage_report.xml
.common-prepare:
extends: .general
only: [merge_requests]
only:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: prepare
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- git lfs install --skip-repo
- git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot"
- export GIT_LFS_SKIP_SMUDGE=0
artifacts:
expire_in: 24 hrs
when: always
......@@ -86,65 +82,247 @@ debian-build:
- sb_branch.txt # Needed to checkout correct branch in build step
- build/*/*/*/*.log # Superbuild log
#-------------------------- precheck job ---------------------------------------
fast-build:
extends: .common
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
#------------------------- prepare & build jobs --------------------------------
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
dependencies: []
## Ubuntu superbuild
ubuntu-superbuild-prepare:
ubuntu-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
ubuntu-superbuild-build:
ubuntu-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
dependencies:
- ubuntu-superbuild-prepare
artifacts:
paths:
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
- build/share/otb
- ubuntu-xdk-prepare
## CentOS superbuild
centos-superbuild-prepare:
centos-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
centos-superbuild-build:
centos-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies:
- centos-superbuild-prepare
- centos-xdk-prepare
## MacOS superbuild
macos-superbuild-prepare:
macos-xdk-prepare:
extends: .common-prepare
tags:
- macos
extends: .common-prepare
before_script:
# No need to install lfs as this machine is persistent
# No need to do git config
# Checkout the expected branch
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
macos-superbuild-build:
macos-xdk-build:
tags:
- macos
extends: .common-build
script:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies:
- macos-xdk-prepare
## Windows
.windows-prepare:
extends: .common-prepare
before_script:
# This override the previous before_script
- set GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q %CI_COMMIT_SHA%
- set GIT_LFS_SKIP_SMUDGE=0
.windows-build:
extends: .common-build
before_script:
- git lfs fetch origin %CI_COMMIT_SHA%
- git checkout -f -q %CI_COMMIT_SHA%
# - Win10
windows-10-prepare:
extends: .windows-prepare
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 xdk 10
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-10-build:
extends: .windows-build
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 otb 10
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-10-prepare
# - Win8.1
windows-8-prepare:
extends: .windows-prepare
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 xdk 8.1
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-8-build:
extends: .windows-build
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 otb 8.1
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-8-prepare
#------------------------- QA related jobs -------------------------------------
ubuntu-xdk-qa-code-coverage:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96"
ctest -V -S CI/main_qa.cmake
-DIMAGE_NAME:string=ubuntu-18.04-llvm-qa
-DQA:BOOL=ON
- ./CI/otb_coverage.sh
- saxon-xslt -o build/ctest_report.xml
build/Testing/`head -n 1 build/Testing/TAG`/Test.xml
CI/ctest2junit.xsl
dependencies:
- ubuntu-xdk-prepare
ubuntu-xdk-qa-static-analysis:
extends: .common-build
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
script:
- find Modules -type f -regextype posix-extended -regex '.*\.(h|hxx)$' -exec dirname '{}' \; |
grep -vE '^Modules/ThirdParty/' |
sort -u > header_directories
- cppcheck -j 8 -q --xml --xml-version=2 --enable=all
--language=c++ --std=c++14 --platform=unix64
--includes-file=header_directories
-i Modules/ThirdParty Modules
2> build/cppcheck_report.xml
after_script: []
dependencies:
- ubuntu-xdk-prepare
ubuntu-xdk-report:
extends: .common
variables:
GIT_DEPTH: ""
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-qa:18.04
stage: report
only:
refs:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
variables:
- $SONAR_OTB_TOKEN
script:
- sonar-scanner -Dproject.settings=sonar-project.properties
-Dsonar.host.url=https://sonar.orfeo-toolbox.org
-Dsonar.login=$SONAR_OTB_TOKEN
`test -z "$CI_COMMIT_TAG" || echo "-Dsonar.projectVersion=$CI_COMMIT_TAG"`
-Dsonar.branch.name=$CI_COMMIT_REF_NAME
after_script: []
dependencies:
- macos-superbuild-prepare
- ubuntu-xdk-qa-code-coverage
- ubuntu-xdk-qa-static-analysis
#---------------------------- Deploy job ---------------------------------------
deploy:
tags:
- deploy
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
- develop@orfeotoolbox/otb
- /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
- git checkout -f -q $CI_COMMIT_SHA
script:
- ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
dependencies:
- ubuntu-xdk-build
- centos-xdk-build
- macos-xdk-build
- windows-8-build
- windows-10-build
release-container:
image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage: deploy
extends: .general
only:
refs:
- tags@orfeotoolbox/otb
variables:
- $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
script:
- curl --request POST
--form token=$K8S_SECRET_RELEASE
--form ref=master
--form variables[OTB_TAG]=$CI_COMMIT_TAG
https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline
......@@ -36,7 +36,6 @@ Once all blocking issues are closed, and the previous steps are done:
* [ ] Software Guide
* [ ] Cookbook
* [ ] Doxygen
* [ ] Application online documentation
* [ ] WordPress page "Home" and "Download" pages
* [ ] Upload OTB source archive to [Zenodo](https://zenodo.org/) to create a unique Digital Object Identifier (DOI)
* [ ] Send email to mailing list to announce the release
......
......@@ -28,6 +28,7 @@ import unittest
import sys
import json
import time
import xml.etree.ElementTree as ET
trace = False
......@@ -42,11 +43,10 @@ def CheckEnvParameters(params):
return False
return True
"""
Handler class to retrieve build informations
"""
class Handler:
# project
# site
# stamp
# name
def __init__ (self):
self.build_dir = ""
self.configure_path = ""
......@@ -75,91 +75,39 @@ class Handler:
if os.path.exists( configure_xml ):
if trace:
print ( configure_xml )
configure_file = open( configure_xml, "r" )
content = configure_file.read()
configure_file.close()
print( content )
self.configure_path = configure_xml
return self.configure_path
print("Could not find the Configure.xml produced by ctest")
return
def GetSite (self , build_dir="" ):
"""
Site is corresponding to the Name field in the xml.
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetSite function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
site_regex = re.compile( "\\bName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)")
site = site_regex.search( content )
if trace:
print (site_regex)
print(site)
if site:
if trace:
print("site value \n" , site.group(1))
self.site = site.group(1)
return self.site
print("Could not retreive site value")
return
return
def GetName (self , build_dir = ""):
"""
This function is looking for the name information in the build tree:
which is BuildName
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetName function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
name_regex = re.compile( "\\bBuildName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
name = name_regex.search( content )
if trace:
print (name_regex)
print( name)
if name:
if trace:
print("name value \n" , name.group(1))
self.name = name.group(1)
return self.name
print("Could not retreive name value")
return
sys.exit(1)
def GetStamp (self , build_dir = "" ):
def ParseConfigureFile(self):
"""
This function is looking for the stamp information in the build tree
Parse the configuration file to get Name, Site and BuildStamp
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetStamp function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
stamp_regex = re.compile( "\\bBuildStamp\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
stamp = stamp_regex.search( content )
# strip the Log section as it can mess up the XML parser
startLog=content.find('<Log>')
endLog=content.rfind('</Log>')
if startLog > 0 and endLog > startLog:
content = content[:(startLog+5)]+content[endLog:]
# parse XML
root = ET.fromstring(content)
if trace:
print( stamp_regex )
print( stamp )
if stamp:
if trace:
print("Stamp value \n" , stamp.group(1))
self.stamp = stamp.group(1)
return self.stamp
print("Could not retreive stamp value")
return
print( root.attrib )
if not 'Name' in root.keys():
print("Can't find site name in Configure.XML")
sys.exit(1)
if not 'BuildName' in root.keys():
print("Can't find build name in Configure.XML")
sys.exit(1)
if not 'BuildStamp' in root.keys():
print("Can't find build stamp in Configure.XML")
sys.exit(1)
self.site = root.get('Name')
self.name = root.get('BuildName')
self.stamp = root.get('BuildStamp')
def GetBuildId (self, **kwargs):
"""
......@@ -180,15 +128,17 @@ class Handler:
if key == "project":
project = value
if ( site == "" or stamp == "" or name == "" or project == ""):
print( "Missing argument for buildid request \
site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
print( "Missing argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
sys.exit(1)
elif trace:
print( "Argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
buildid_api = "/api/v1/getbuildid.php?"
buildid_params = urllib.parse.urlencode({'project': project, 'site': site, 'stamp': stamp , 'name': name})
full_url = self.url + buildid_api + buildid_params
if trace:
print("full_url: "+full_url)
nb_try = 6
max_retry = 11
nb_try = max_retry
build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
while nb_try:
response = urllib.request.urlopen(full_url).read().decode()
......@@ -198,8 +148,8 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
nb_try -= 1
if buildid or (nb_try == 0):
break
print("No build id, retry ...")
time.sleep(60)
print("No build id, retry "+str(max_retry-nb_try)+"/"+str(max_retry)+" ...")
time.sleep(30)
if buildid:
self.buildid = buildid.group(1)
if trace:
......@@ -207,7 +157,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return buildid.group(1)
else:
print("Error in recovering buildid")
return
sys.exit(1)
def GetBuildUrl (self , buildid = "" ):
"""
......@@ -249,6 +199,34 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
errors = "Errors occur during tests"
return ( state , errors)
def GetReturnValue(self, logfile):
fd = open(logfile)
content = fd.readlines()[0]
fd.close()
return int(content.strip("\n"))
def GetLogStatus(self, logdir):
"""
This function returns the log status of a build as a pair 'state' + 'errors'
"""
configure_rv = os.path.join(logdir, "configure_return_value_log.txt")
build_rv = os.path.join(logdir, "build_return_value_log.txt")
test_rv = os.path.join(logdir, "test_return_value_log.txt")
if os.path.exists( configure_rv ):
if (self.GetReturnValue(configure_rv) != 0):
return ( 'failed' , 'Configure failed')
else:
return ( 'failed' , 'Configure not run')
if os.path.exists( build_rv ):
if (self.GetReturnValue(build_rv) != 0):
return ( 'failed' , 'Build failed')
else:
return ( 'failed' , 'Build not run')
if os.path.exists( test_rv ):
if (self.GetReturnValue(test_rv) != 0):
return ( 'failed' , 'Tests failed')
return ('success', '')
"""
This script aims only at recovering the build url
It uses environment variables setup by Gitlab Runner as default:
......@@ -262,11 +240,11 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
cdash_handler.py commit_sha1 project_id project_directory token ref_name
"""
if __name__ == "__main__":
if trace:
print(sys.argv)
if ( len(sys.argv) < 6 and len(sys.argv) > 1 ):
print("Usage : "+sys.argv[0]+" commit_sha1 project_id project_directory token ref_name")
sys.exit(1)
if trace:
print (sys.argv)
if ( len(sys.argv) >= 6):
sha1 = sys.argv[1]
proj = sys.argv[2]
......@@ -274,33 +252,36 @@ if __name__ == "__main__":
token = sys.argv[4]
refn = sys.argv[5]
else:
if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'K8S_SECRET_API_TOKEN', 'CI_COMMIT_REF_NAME']):
if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'CI_COMMIT_REF_NAME']):
sys.exit(1)
sha1 = os.environ['CI_COMMIT_SHA']
proj = os.environ['CI_PROJECT_ID']
pdir = os.environ['CI_PROJECT_DIR']
token = os.environ['K8S_SECRET_API_TOKEN']
if 'CI_MERGE_REQUEST_REF_PATH' in os.environ.keys():
refn = os.environ['CI_MERGE_REQUEST_REF_PATH']
else:
refn = os.environ['CI_COMMIT_REF_NAME']
if CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
token = os.environ['K8S_SECRET_API_TOKEN']
else:
token = None
handler = Handler()
build_dir = os.path.join( pdir , "build/")
if trace:
print("build_dir is: " + build_dir)
handler.build_dir = build_dir
handler.GetSite()
handler.GetName()
handler.GetStamp()