Merge branch 'ci_alpha_fixes' into 'develop'

CI Fixes episode N

See merge request !536

.gitlab-ci.yml

@@ -34,26 +34,11 @@ stages:
 .common:
   extends: .general
   before_script:
-    - git lfs install --skip-repo
     # Provision efficiently the local LFS cache before checkout
     - git lfs fetch origin $CI_COMMIT_SHA
-    # Checkout the expected branch
     - git checkout -f -q $CI_COMMIT_SHA
   after_script:
-    - python3 CI/cdash_handler.py
-
-fast-build:
-  extends: .common
-  only: [merge_requests, branches]
-  stage: precheck
-  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
-  before_script:
-    - export GIT_LFS_SKIP_SMUDGE=1
-    - git checkout -f -q $CI_COMMIT_SHA
-    - python3 CI/check_twin_pipelines.py
-  script:
-    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
-    - ccache -s
+    - python3 -u CI/cdash_handler.py
 
 .common-build:
   extends: .common
@@ -74,12 +59,6 @@ fast-build:
       - build_packages/OTB-*.run
       - build_packages/OTB-*.zip
 
-debian-build:
-  extends: .common-build
-  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
-  script:
-    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
-
 .common-prepare:
   extends: .general
   only:
@@ -88,7 +67,6 @@ debian-build:
     - /^release-[0-9]+\.[0-9]+$/
   stage: prepare
   before_script:
-    - git lfs install --skip-repo
     - export GIT_LFS_SKIP_SMUDGE=1
     - git checkout -f -q $CI_COMMIT_SHA
     - export GIT_LFS_SKIP_SMUDGE=0
@@ -99,6 +77,27 @@ debian-build:
       - sb_branch.txt # Needed to checkout correct branch in build step
       - build/*/*/*/*.log # Superbuild log
 
+#-------------------------- precheck job ---------------------------------------
+fast-build:
+  extends: .common
+  only: [merge_requests, branches]
+  stage: precheck
+  image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
+  before_script:
+    - export GIT_LFS_SKIP_SMUDGE=1
+    - git checkout -f -q $CI_COMMIT_SHA
+    - python3 CI/check_twin_pipelines.py
+  script:
+    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
+    - ccache -s
+
+#------------------------- prepare & build jobs --------------------------------
+debian-build:
+  extends: .common-build
+  image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
+  script:
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
+
 ## Ubuntu superbuild
 ubuntu-xdk-prepare:
   extends: .common-prepare
@@ -111,7 +110,7 @@ ubuntu-xdk-build:
   image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
   script:
     - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
-    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
   dependencies:
     - ubuntu-xdk-prepare
 
@@ -127,20 +126,15 @@ centos-xdk-build:
   image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
   script:
     - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
-    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
+    - xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
   dependencies:
     - centos-xdk-prepare
 
 ## MacOS superbuild
 macos-xdk-prepare:
+  extends: .common-prepare
   tags:
     - macos
-  extends: .common-prepare
-  before_script:
-    # No need to install lfs as this machine is persistent
-    - export GIT_LFS_SKIP_SMUDGE=1
-    - git checkout -f -q $CI_COMMIT_SHA
-    - export GIT_LFS_SKIP_SMUDGE=0
   script:
     - ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
 
@@ -150,7 +144,7 @@ macos-xdk-build:
   extends: .common-build
   script:
     - ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
-    - ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
+    - ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
   dependencies:
     - macos-xdk-prepare
 
@@ -217,9 +211,7 @@ windows-8-build:
   dependencies:
     - windows-8-prepare
 
-    
-# Deploy job
-
+#---------------------------- Deploy job ---------------------------------------
 deploy:
   tags:
     - deploy
@@ -227,13 +219,11 @@ deploy:
   stage: deploy
   extends: .general
   only:
-    - develop
-    - /^release-[0-9]+\.[0-9]+$/
+    - develop@orfeotoolbox/otb
+    - /^release-[0-9]+\.[0-9]+$/@orfeotoolbox/otb
   before_script:
-    - git lfs install --skip-repo
     # Provision efficiently the local LFS cache before checkout
     - git lfs fetch origin $CI_COMMIT_SHA
-    # Checkout the expected branch
     - git checkout -f -q $CI_COMMIT_SHA
   script:
     - ./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
@@ -242,4 +232,16 @@ deploy:
     - centos-xdk-build
     - macos-xdk-build
     - windows-8-build
-    - windows-10-build
\ No newline at end of file
+    - windows-10-build
+
+release-container:
+  image: $BUILD_IMAGE_REGISTRY/otb-alpine:3.7
+  stage: deploy
+  extends: .general
+  only:
+    refs:
+      - tags@orfeotoolbox/otb
+    variables:
+      - $CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
+  script:
+    - "curl --request POST --form token=$K8S_SECRET_RELEASE --form ref=master --form variables[OTB_TAG]=$CI_COMMIT_TAG https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline"

CI/cdash_handler.py

@@ -28,6 +28,7 @@ import unittest
 import sys
 import json
 import time
+import xml.etree.ElementTree as ET
 
 
 trace = False
@@ -42,11 +43,10 @@ def CheckEnvParameters(params):
       return False
   return True
 
+"""
+Handler class to retrieve build informations
+"""
 class Handler:
-# project
-# site
-# stamp
-# name
   def __init__ (self):
     self.build_dir = ""
     self.configure_path = ""
@@ -75,91 +75,39 @@ class Handler:
     if os.path.exists( configure_xml ):
       if trace:
         print ( configure_xml )
-        configure_file = open( configure_xml, "r" )
-        content = configure_file.read()
-        configure_file.close()
-        print( content )
       self.configure_path = configure_xml
       return self.configure_path
     print("Could not find the Configure.xml produced by ctest")
-    return
-
-  def GetSite (self , build_dir="" ):
-    """
-    Site is corresponding to the Name field in the xml.
-    """
-    if ( build_dir == ""):
-      build_dir = self.build_dir
-    if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
-      print ("Error in GetSite function, could not find Configure.xml")
-      return
-    configure_file = open( self.configure_path, "r" )
-    content = configure_file.read()
-    configure_file.close()
-    site_regex = re.compile( "\\bName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)")
-    site = site_regex.search( content )
-    if trace:
-      print (site_regex)
-      print(site)
-    if site:
-      if trace:
-        print("site value \n" , site.group(1))
-      self.site = site.group(1)
-      return self.site
-    print("Could not retreive site value")
-    return
-    return 
-
-  def GetName (self , build_dir = ""):
-    """
-    This function is looking for the name information in the build tree: 
-    which is BuildName
-    """
-    if ( build_dir == ""):
-      build_dir = self.build_dir
-    if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
-      print ("Error in GetName function, could not find Configure.xml")
-      return
-    configure_file = open( self.configure_path, "r" )
-    content = configure_file.read()
-    configure_file.close()
-    name_regex = re.compile( "\\bBuildName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
-    name = name_regex.search( content )
-    if trace:
-      print (name_regex)
-      print( name)
-    if name:
-      if trace:
-        print("name value \n" , name.group(1))
-      self.name = name.group(1)
-      return self.name
-    print("Could not retreive name value")
-    return
+    sys.exit(1)
 
-  def GetStamp (self , build_dir = "" ):
+  def ParseConfigureFile(self):
     """
-    This function is looking for the stamp information in the build tree
+    Parse the configuration file to get Name, Site and BuildStamp
     """
-    if ( build_dir == ""):
-      build_dir = self.build_dir
-    if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
-      print ("Error in GetStamp function, could not find Configure.xml")
-      return
     configure_file = open( self.configure_path, "r" )
     content = configure_file.read()
     configure_file.close()
-    stamp_regex = re.compile( "\\bBuildStamp\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
-    stamp = stamp_regex.search( content )
+    # strip the Log section as it can mess up the XML parser
+    startLog=content.find('<Log>')
+    endLog=content.rfind('</Log>')
+    if startLog > 0 and endLog > startLog:
+      content = content[:(startLog+5)]+content[endLog:]
+    # parse XML
+    root = ET.fromstring(content)
     if trace:
-      print( stamp_regex )
-      print( stamp )
-    if stamp:
-      if trace:
-        print("Stamp value \n" , stamp.group(1))
-      self.stamp = stamp.group(1)
-      return self.stamp
-    print("Could not retreive stamp value")
-    return
+      print( root.attrib )
+    if not 'Name' in root.keys():
+      print("Can't find site name in Configure.XML")
+      sys.exit(1)
+    if not 'BuildName' in root.keys():
+      print("Can't find build name in Configure.XML")
+      sys.exit(1)
+    if not 'BuildStamp' in root.keys():
+      print("Can't find build stamp in Configure.XML")
+      sys.exit(1)
+    self.site = root.get('Name')
+    self.name = root.get('BuildName')
+    self.stamp = root.get('BuildStamp')
 
   def GetBuildId (self, **kwargs):
     """
@@ -180,15 +128,17 @@ class Handler:
       if key == "project":
         project = value
     if ( site == "" or stamp == "" or name == "" or project == ""):
-      print( "Missing argument for buildid request \
-site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
-      return
+      print( "Missing argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
+      sys.exit(1)
+    elif trace:
+      print( "Argument for buildid request site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
     buildid_api = "/api/v1/getbuildid.php?"
     buildid_params = urllib.parse.urlencode({'project': project, 'site': site, 'stamp': stamp , 'name': name})
     full_url = self.url + buildid_api + buildid_params
     if trace:
       print("full_url: "+full_url)
-    nb_try = 6
+    max_retry = 11
+    nb_try = max_retry
     build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
     while nb_try:
       response = urllib.request.urlopen(full_url).read().decode()
@@ -198,8 +148,8 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
       nb_try -= 1
       if buildid or (nb_try == 0):
         break
-      print("No build id, retry ...")
-      time.sleep(60)
+      print("No build id, retry "+str(max_retry-nb_try)+"/"+str(max_retry)+" ...")
+      time.sleep(30)
     if buildid:
       self.buildid = buildid.group(1)
       if trace:
@@ -207,7 +157,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
       return buildid.group(1)
     else:
       print("Error in recovering buildid")
-      return
+      sys.exit(1)
 
   def GetBuildUrl (self , buildid = "" ):
     """
@@ -290,11 +240,11 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
     cdash_handler.py commit_sha1  project_id  project_directory  token  ref_name
 """
 if __name__ == "__main__":
+  if trace:
+    print(sys.argv)
   if ( len(sys.argv) < 6 and len(sys.argv) > 1 ):
     print("Usage : "+sys.argv[0]+" commit_sha1 project_id project_directory token ref_name")
     sys.exit(1)
-  if trace:
-    print (sys.argv)
   if ( len(sys.argv) >= 6):
     sha1 = sys.argv[1]
     proj = sys.argv[2]
@@ -302,24 +252,26 @@ if __name__ == "__main__":
     token = sys.argv[4]
     refn = sys.argv[5]
   else:
-    if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'K8S_SECRET_API_TOKEN', 'CI_COMMIT_REF_NAME']):
+    if not CheckEnvParameters(['CI_COMMIT_SHA', 'CI_PROJECT_ID', 'CI_PROJECT_DIR', 'CI_COMMIT_REF_NAME']):
       sys.exit(1)
     sha1 = os.environ['CI_COMMIT_SHA']
     proj = os.environ['CI_PROJECT_ID']
     pdir = os.environ['CI_PROJECT_DIR']
-    token = os.environ['K8S_SECRET_API_TOKEN']
     if 'CI_MERGE_REQUEST_REF_PATH' in os.environ.keys():
       refn = os.environ['CI_MERGE_REQUEST_REF_PATH']
     else:
       refn = os.environ['CI_COMMIT_REF_NAME']
+    if CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
+      token = os.environ['K8S_SECRET_API_TOKEN']
+    else:
+      token = None
   handler = Handler()
   build_dir = os.path.join( pdir , "build/")
   if trace:
     print("build_dir is: " + build_dir)
   handler.build_dir = build_dir
-  handler.GetSite()
-  handler.GetName()
-  handler.GetStamp()
+  handler.GetConfigureFile()
+  handler.ParseConfigureFile()
   if handler.GetBuildId() is None:
     cdash_url = "https://cdash.orfeo-toolbox.org"
     state = 'failed'
@@ -327,8 +279,9 @@ if __name__ == "__main__":
   else:
     cdash_url = handler.GetBuildUrl()
     ( state , error ) = handler.GetLogStatus( os.path.join( pdir , "log") )
-  if trace:
-    print ( "cdash_url is: " + cdash_url )
+  print("CDash build URL : "+cdash_url)
+  if token is None:
+    sys.exit(0)
   gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
   gitlab_url += proj + "/statuses/" + sha1
   params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\

CI/check_twin_pipelines.py

@@ -24,6 +24,7 @@ import urllib.parse
 import json
 import re
 import time
+import sys
 
 """
 Send a request to Gitlab and return the answer
@@ -65,9 +66,10 @@ if __name__ == "__main__":
   # are we in a merge_request pipeline ?
   if 'CI_MERGE_REQUEST_IID' in env.keys():
     if not CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
-      print("Make sure you have set a valid acces token for Gitlab API")
-      print("The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'")
-      sys.exit(1)
+      print("WARNING: Make sure you have set a valid acces token for Gitlab API." \
+        + "The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'" \
+        + "Without this token, some feature of the CI platform will be disabled.")
+      sys.exit(0)
     if not CheckEnvParameters(['CI_PROJECT_ID','CI_PIPELINE_ID']):
       sys.exit(1)
     mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_API_TOKEN'])

Packaging/detect_using_file_command.cmake

@@ -115,10 +115,10 @@ function(detect_using_file_command input_file result_type result_dir)
   # Patch file command returning shared-oject for executable on Linux when -PIE is used.
   if( LINUX )
     get_filename_component( input_file_DIR ${input_file} DIRECTORY )
-    get_filename_component( input_file_dIR ${input_file_DIR} NAME )
-    string( TOLOWER "${input_file_DIR}" input_file_DIR )
-    if( ${input_file_DIR} MATCHES "bin" )
-      message( WARNING "${input_file} detected as shared-object." )
+    get_filename_component( dir_name ${input_file_DIR} NAME )
+    string( TOLOWER "${dir_name}" dir_name_l )
+    if( "${dir_name_l}" STREQUAL "bin" )
+      message( STATUS "${input_file} detected as shared-object, but processed as executable" )
       set(detected_type PROGRAMS)
       set(detected_dir bin)
     endif()