Merge branch 'ci_basic_step' into 'develop'

CI: add fast precheck step

See merge request orfeotoolbox/otb!465

.gitlab-ci.yml

@@ -29,6 +29,7 @@ after_script:
   - python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH
 
 stages:
+  - precheck
   - prepare
   - build
 
@@ -39,6 +40,19 @@ stages:
       - runner_system_failure
       - stuck_or_timeout_failure
 
+fast-build:
+  extends: .general
+  only: [merge_requests, branches]
+  stage: precheck
+  image: $CI_REGISTRY/gpasero/otb/otb-install-ubuntu-native
+  before_script:
+    - export GIT_LFS_SKIP_SMUDGE=1
+    - git checkout $CI_COMMIT_REF_NAME
+    - python3 CI/check_twin_pipelines.py
+  script:
+    - ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
+    - ccache -s
+
 debian-build:
   extends: .general
   only: [merge_requests]
@@ -59,15 +73,10 @@ debian-build:
       - build/Documentation/Cookbook/latex/CookBook-*.pdf
       - build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
 
-# This is needed to have only one pipeline in a merge request context
 ubuntu-llvm:
   only: [merge_requests]
   extends: .common-build
 
-ubuntu-llvm-wip:
-  except: [merge_requests]
-  extends: .common-build
-
 superbuild-prepare:
   only: [merge_requests]
   extends: .general

CI/check_twin_pipelines.py

+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import urllib.request
+import urllib.parse
+import json
+import re
+import time
+
+"""
+Send a request to Gitlab and return the answer
+The request parameter is added after `project/:id/`
+WARNING: when data is given, the request will be a POST
+Otherwise, it is a GET
+"""
+def GitlabRequest(request, project=53, data=None, token=''):
+  gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
+  gitlab_url+= str(project) + '/' + request
+  params = None
+  myHeader = {}
+  if not data is None:
+    params = urllib.parse.urlencode(data).encode('ascii')
+  if token:
+    myHeader = {'PRIVATE-TOKEN':token}
+  gitlab_request = urllib.request.Request(gitlab_url, data=params, headers=myHeader)
+  res = urllib.request.urlopen(gitlab_request)
+  return json.loads(res.read().decode())
+
+"""
+Check needed environment parameters
+"""
+def CheckEnvParameters(params):
+  for p in params:
+    if not p in os.environ.keys():
+      print("Missing environment variable '"+p+"'")
+      return False
+  return True
+
+"""
+Check for any duplicated twin pipeline and cancel it
+"""
+if __name__ == "__main__":
+  if not CheckEnvParameters(['CI_COMMIT_SHA']):
+    sys.exit(1)
+  env = os.environ
+  sha1 = env['CI_COMMIT_SHA']
+  # are we in a merge_request pipeline ?
+  if 'CI_MERGE_REQUEST_IID' in env.keys():
+    if not CheckEnvParameters(['K8S_SECRET_TWIN_PIPELINE','CI_PROJECT_ID','CI_PIPELINE_ID']):
+      sys.exit(1)
+    mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_TWIN_PIPELINE'])
+    wip_regex = re.compile("^[Ww][Ii][Pp]:")
+    # is it a "WIP" merge request ?
+    if wip_regex.search(mrInfo["title"]):
+      # Yes: cancel the current pipeline
+      print("Cancel current pipeline "+env['CI_PIPELINE_ID'])
+      GitlabRequest('pipelines/'+env['CI_PIPELINE_ID']+'/cancel', data={}, \
+        project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
+      time.sleep(180)
+      print("Error: this pipeline should have been canceled")
+      sys.exit(1)
+    else:
+      # No: cancel any previous "normal" pipeline on the same SHA1
+      jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
+      for item in jres:
+        if item["id"] < int(env['CI_PIPELINE_ID']) and item["status"] == "running":
+          print("Cancel pipeline "+str(item["id"]))
+          jres2 = GitlabRequest('pipelines/'+str(item["id"])+'/cancel', data={}, \
+            project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
+

CI/main_ci.cmake

@@ -68,6 +68,11 @@ if(NOT DEFINED IMAGE_NAME)
 endif()
 set (CTEST_SITE "${IMAGE_NAME}")
 
+# Detect "skip testing"
+if(DEFINED ENV{CI_SKIP_TESTING})
+  set(ci_skip_testing 1)
+endif()
+
 # Directory variable
 set (CTEST_SOURCE_DIRECTORY "${OTB_SOURCE_DIR}")
 if(BUILD_DIR)
@@ -130,10 +135,15 @@ if ( NOT _build_rv EQUAL 0 )
   message( SEND_ERROR "An error occurs during ctest_build.")
 endif()
 
-ctest_test(PARALLEL_LEVEL 8
-           RETURN_VALUE _test_rv
-           CAPTURE_CMAKE_ERROR _test_error
-           )
+if(ci_skip_testing)
+  message(STATUS "Skip testing")
+  set(_test_rv 0)
+else()
+  ctest_test(PARALLEL_LEVEL 8
+             RETURN_VALUE _test_rv
+             CAPTURE_CMAKE_ERROR _test_error
+             )
+endif()
 
 if ( NOT _test_rv EQUAL 0 )
   message( SEND_ERROR "An error occurs during ctest_test.")

CI/ubuntu-18.04-fast.cmake

+#
+# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
+#
+# This file is part of Orfeo Toolbox
+#
+#     https://www.orfeo-toolbox.org/
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Configuration options for ubuntu-18.04-fast
+
+set(site_option
+"opencv_INCLUDE_DIR:PATH=/usr/include
+CMAKE_C_COMPILER:STRING=clang
+CMAKE_CXX_COMPILER:STRING=clang++
+CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld
+CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld
+CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld
+CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
+CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
+OTB_USE_SHARK:BOOL=OFF
+BUILD_EXAMPLES:BOOL=OFF")
+
+set(ci_skip_testing ON)