Commit 906e7231 authored by Antoine Regimbeau's avatar Antoine Regimbeau

Merge branch 'develop' into superbuild_data

parents f556bccd 0df44b31
Pipeline #1325 passed with stages
in 2 minutes and 16 seconds
......@@ -5,11 +5,12 @@
# But Git fails to manage efficiently numerous LFS data.
# We have to use directly git-lfs wrapping commands to have an
# efficient cloning step.
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
# Furthermore, Git-LFS and Gitlab sufer a bug preventing usage of
# GIT_STRATEGY=fetch (https://gitlab.com/gitlab-org/gitlab-runner/issues/3318)
variables:
BUILD_IMAGE_REGISTRY: $CI_REGISTRY/gbonnefille/otb-build-env
BUILD_IMAGE_REGISTRY: $CI_REGISTRY/orfeotoolbox/otb-build-env
GIT_CLONE_PATH: $CI_BUILDS_DIR/otb
DOCKER_DRIVER: overlay2
GIT_DEPTH: "3"
# Disable automatic checkout to let us fetch LFS before
......@@ -29,6 +30,8 @@ after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH
stages:
- precheck
- prepare
- build
.general:
......@@ -38,36 +41,92 @@ stages:
- runner_system_failure
- stuck_or_timeout_failure
native-build:
fast-build:
extends: .general
only: [merge_requests]
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
only: [merge_requests, branches]
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout $CI_COMMIT_REF_NAME
- python3 CI/check_twin_pipelines.py
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
- ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
- ccache -s
debian-build:
.common-build:
extends: .general
only: [merge_requests]
stage: build
dependencies: []
debian-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-build:
ubuntu-llvm:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm
artifacts:
paths:
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
.common-prepare:
extends: .general
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-shark:18.04
only: [merge_requests]
stage: prepare
before_script:
# This override the previous before_script
- git checkout $CI_COMMIT_REF_NAME
# We are now doing the git-lfs install
# This is done after the checkout so we avoid downloading Data
# But we need it to upload the archive
- git-lfs install
- git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot"
- eval $(ssh-agent -s)
- ssh-add <(echo "$K8S_SECRET_SSH")
# This is for debug, we are checking the owner of the ssh key
- ssh -o StrictHostKeyChecking=no -T git@gitlab.orfeo-toolbox.org
after_script:
- echo "Nothing to do for after_script"
artifacts:
expire_in: 24 hrs
paths:
# This recovers logs from superbuild build
- build/*/*/*/*.log
- sb_branch.txt
ubuntu-superbuild-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-shark
- ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
centos-superbuild-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- ctest -V -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
# This is needed to have only one pipeline in a merge request context
ubuntu-llvm:
only: [merge_requests]
ubuntu-superbuild-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
dependencies:
- ubuntu-superbuild-prepare
ubuntu-llvm-wip:
except: [merge_requests]
centos-superbuild-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies:
- centos-superbuild-prepare
......@@ -45,3 +45,4 @@ The copyright owner is *COPYRIGHT OWNER (OR OWNER'S AGENT)* and has signed the O
- The feature branch is (reasonably) up-to-date with the base branch
- Dashboard is green
- Copyright owner has signed the ORFEO ToolBox Contributor License Agreement
- Optionally, run `git diff develop... -U0 --no-color | clang-format-diff.py -p1 -i` on latest changes and commit
......@@ -26,6 +26,7 @@ import re
import unittest
import sys
import json
import time
trace = False
......@@ -168,7 +169,7 @@ class Handler:
if key == "project":
project = value
if ( site == "" or stamp == "" or name == "" or project == ""):
print( "Not enougth argument given for buildid request \
print( "Missing argument for buildid request \
site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
buildid_api = "/api/v1/getbuildid.php?"
......@@ -176,11 +177,18 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
full_url = self.url + buildid_api + buildid_params
if trace:
print("full_url: "+full_url)
response = urllib.request.urlopen(full_url).read().decode()
if trace:
print ( "response: " + response )
nb_try = 6
build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
buildid = build_id_regex.search( response )
while nb_try:
response = urllib.request.urlopen(full_url).read().decode()
if trace:
print ( "response: " + response )
buildid = build_id_regex.search( response )
nb_try -= 1
if buildid or (nb_try == 0):
break
print("No build id, retry ...")
time.sleep(60)
if buildid:
self.buildid = buildid.group(1)
if trace:
......@@ -198,7 +206,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Not enougth argument given to build url")
print( "Missing argument to build url")
return
build_url = self.url
build_url +="/buildSummary.php?"
......@@ -212,7 +220,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Not enougth argument given to build Status")
print( "Missing argument to build Status")
return
full_url = self.url + "/api/v1/buildSummary.php?buildid=" + buildid
response = urllib.request.urlopen(full_url).read().decode()
......@@ -251,10 +259,13 @@ if __name__ == "__main__":
handler.GetSite()
handler.GetName()
handler.GetStamp()
handler.GetBuildId()
# handler.buildid="1"
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetBuildStatus()
if handler.GetBuildId() is None:
cdash_url = "https://cdash.orfeo-toolbox.org"
state = 'failed'
error = "Failed to get build id"
else:
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetBuildStatus()
if trace:
print ( "cdash_url is: " + cdash_url )
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
......
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import urllib.request
import urllib.parse
import json
import re
import time
"""
Send a request to Gitlab and return the answer
The request parameter is added after `project/:id/`
WARNING: when data is given, the request will be a POST
Otherwise, it is a GET
"""
def GitlabRequest(request, project=53, data=None, token=''):
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url+= str(project) + '/' + request
params = None
myHeader = {}
if not data is None:
params = urllib.parse.urlencode(data).encode('ascii')
if token:
myHeader = {'PRIVATE-TOKEN':token}
gitlab_request = urllib.request.Request(gitlab_url, data=params, headers=myHeader)
res = urllib.request.urlopen(gitlab_request)
return json.loads(res.read().decode())
"""
Check needed environment parameters
"""
def CheckEnvParameters(params):
for p in params:
if not p in os.environ.keys():
print("Missing environment variable '"+p+"'")
return False
return True
"""
Check for any duplicated twin pipeline and cancel it
"""
if __name__ == "__main__":
if not CheckEnvParameters(['CI_COMMIT_SHA']):
sys.exit(1)
env = os.environ
sha1 = env['CI_COMMIT_SHA']
# are we in a merge_request pipeline ?
if 'CI_MERGE_REQUEST_IID' in env.keys():
if not CheckEnvParameters(['K8S_SECRET_TWIN_PIPELINE','CI_PROJECT_ID','CI_PIPELINE_ID']):
sys.exit(1)
mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_TWIN_PIPELINE'])
wip_regex = re.compile("^[Ww][Ii][Pp]:")
# is it a "WIP" merge request ?
if wip_regex.search(mrInfo["title"]):
# Yes: cancel the current pipeline
print("Cancel current pipeline "+env['CI_PIPELINE_ID'])
GitlabRequest('pipelines/'+env['CI_PIPELINE_ID']+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
time.sleep(180)
print("Error: this pipeline should have been canceled")
sys.exit(1)
else:
# No: cancel any previous "normal" pipeline on the same SHA1
jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
for item in jres:
if item["id"] < int(env['CI_PIPELINE_ID']) and item["status"] == "running":
print("Cancel pipeline "+str(item["id"]))
jres2 = GitlabRequest('pipelines/'+str(item["id"])+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
......@@ -19,13 +19,13 @@
#
# This script is a prototype for the future CI, it may evolve rapidly in a near future
#This file set the following variable :
#This file set the following variable :
# * otb_build_project_option
# * otb_use_option
# * otb_wrap_option
# * CONFIGURE_OPTIONS
set (otb_build_project_option
set (otb_build_project_option
"BUILD_COOKBOOK:BOOL=OFF
BUILD_EXAMPLES:BOOL=ON
BUILD_SHARED_LIBS:BOOL=ON
......@@ -40,7 +40,7 @@ OTB_USE_GLUT:BOOL=ON
OTB_USE_GSL:BOOL=ON
OTB_USE_LIBKML:BOOL=ON
OTB_USE_LIBSVM:BOOL=ON
OTB_USE_MPI:BOOL=ON
OTB_USE_MPI:BOOL=OFF
OTB_USE_MUPARSER:BOOL=ON
OTB_USE_MUPARSERX:BOOL=ON
OTB_USE_OPENCV:BOOL=ON
......@@ -51,12 +51,12 @@ OTB_USE_QWT:BOOL=ON
OTB_USE_SHARK:BOOL=ON
OTB_USE_SIFTFAST:BOOL=ON
OTB_USE_SPTW:BOOL=ON
OTB_USE_SSE_FLAGS:BOOL=ON")
OTB_USE_SSE_FLAGS:BOOL=ON
OTB_MPIEXEC_OPT:STRING=--allow-run-as-root")
set (otb_wrap_option
"OTB_WRAP_JAVA:BOOL=ON
OTB_WRAP_PYTHON:BOOL=ON
OTB_WRAP_PYTHON3:BOOL=ON")
OTB_WRAP_PYTHON:BOOL=ON")
set (otb_data_option
"OTB_DATA_USE_LARGEINPUT:BOOL=OFF
......
......@@ -17,31 +17,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script is for the superbuild build on the CI platform
otb_module_test()
# Find the build name and CI profile
macro( set_dash_build_name )
set(ci_profile wip)
set(OTBUrbanAreaTests
otbUrbanAreaTestDriver.cxx
otbUrbanAreaDetectionImageFilter.cxx
)
set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
set(ci_mr_iid "$ENV{CI_MERGE_REQUEST_IID}")
add_executable(otbUrbanAreaTestDriver ${OTBUrbanAreaTests})
target_link_libraries(otbUrbanAreaTestDriver ${OTBUrbanArea-Test_LIBRARIES})
otb_module_target_label(otbUrbanAreaTestDriver)
set(ci_pipeline_id "$ENV{CI_PIPELINE_ID}")
set(ci_job_id "$ENV{CI_JOB_ID}")
# Tests Declaration
set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
set (CTEST_BUILD_NAME ${ci_short_sha})
otb_add_test(NAME raTvUrbanAreaDetectionImageFilter COMMAND otbUrbanAreaTestDriver
--compare-image ${EPSILON_8} ${BASELINE}/raTvUrbanAreaDetection.png
${TEMP}/raTvUrbanAreaDetection.png
otbUrbanAreaDetectionImageFilter
${INPUTDATA}/qb_RoadExtract2sub200x200.tif
${TEMP}/raTvUrbanAreaDetection.png
3 # red
2 # green
4 # nir
0.8 # threshold NDVI+NDWI
0.005 # threshold EdgeDensity
)
if(ci_mr_source AND ci_mr_target AND ci_mr_iid)
set (CTEST_BUILD_NAME "${ci_mr_source} (MR ${ci_mr_iid}) P-${ci_pipeline_id}")
set(ci_profile mr)
elseif(ci_ref_name)
set (CTEST_BUILD_NAME "${ci_ref_name} P-${ci_pipeline_id}")
if("${ci_ref_name}" STREQUAL "develop")
set(ci_profile develop)
elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$")
set(ci_profile release)
endif()
endif()
endmacro()
......@@ -18,6 +18,8 @@
# limitations under the License.
#
include( "${CMAKE_CURRENT_LIST_DIR}/macros.cmake" )
# This script is a prototype for the future CI, it may evolve rapidly in a near future
get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
set (ENV{LANG} "C") # Only ascii output
......@@ -26,33 +28,49 @@ set (ENV{LANG} "C") # Only ascii output
set (CTEST_BUILD_CONFIGURATION "Release")
set (CTEST_CMAKE_GENERATOR "Ninja")
# detect short sha
if(NOT DEFINED ENV{CI_COMMIT_SHORT_SHA})
execute_process(COMMAND git log -1 --pretty=format:%h