Commit 3f8b4fb8 authored by Guillaume Pasero's avatar Guillaume Pasero

MRG: Synchronize with branch develop

parents c39c8d99 67881ae4
Pipeline #922 failed with stages
in 29 minutes and 35 seconds
......@@ -25,6 +25,9 @@ before_script:
# Checkout the expected branch
- git checkout $CI_COMMIT_REF_NAME
after_script:
- python3 CI/cdash_handler.py $CI_COMMIT_SHA $CI_PROJECT_ID $CI_PROJECT_DIR $K8S_SECRET_CDASH
stages:
- build
......@@ -41,7 +44,7 @@ native-build:
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
debian-build:
extends: .general
......@@ -49,20 +52,22 @@ debian-build:
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-build:
extends: .general
stage: build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-shark:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-shark
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-shark
build:ubuntu-llvm:
# This is needed to have only one pipeline in a merge request context
ubuntu-llvm:
only: [merge_requests]
extends: .common-build
build:ubuntu-llvm-wip:
ubuntu-llvm-wip:
except: [merge_requests]
extends: .common-build
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os.path
import urllib.request
import urllib.parse
import glob
import re
import unittest
import sys
import json
trace = False
class Handler:
# project
# site
# stamp
# name
def __init__ (self):
self.build_dir = ""
self.configure_path = ""
self.url = "https://cdash.orfeo-toolbox.org"
self.project = "OTB"
self.site = ""
self.stamp = ""
self.name = ""
self.buildid = ""
def GetConfigureFile (self , build_dir = "" ):
if build_dir == "":
build_dir = self.build_dir
build_dir = os.path.normpath(build_dir)
test_path = os.path.join( build_dir , "Testing/")
list_folder = os.listdir(test_path)
if trace:
print(list_folder)
configure_xml = ""
for folder in list_folder:
if folder == "Temporary" or folder == "TAG":
pass
configure_xml = os.path.join( test_path , folder , "Configure.xml" )
if os.path.exists( configure_xml ):
break
if os.path.exists( configure_xml ):
if trace:
print ( configure_xml )
configure_file = open( configure_xml, "r" )
content = configure_file.read()
configure_file.close()
print( content )
self.configure_path = configure_xml
return self.configure_path
print("Could not find the Configure.xml produced by ctest")
return
def GetSite (self , build_dir="" ):
"""
Site is corresponding to the Name field in the xml.
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetSite function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
site_regex = re.compile( "\\bName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)")
site = site_regex.search( content )
if trace:
print (site_regex)
print(site)
if site:
if trace:
print("site value \n" , site.group(1))
self.site = site.group(1)
return self.site
print("Could not retreive site value")
return
return
def GetName (self , build_dir = ""):
"""
This function is looking for the name information in the build tree:
which is BuildName
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetName function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
name_regex = re.compile( "\\bBuildName\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
name = name_regex.search( content )
if trace:
print (name_regex)
print( name)
if name:
if trace:
print("name value \n" , name.group(1))
self.name = name.group(1)
return self.name
print("Could not retreive name value")
return
def GetStamp (self , build_dir = "" ):
"""
This function is looking for the stamp information in the build tree
"""
if ( build_dir == ""):
build_dir = self.build_dir
if self.configure_path == "" and not self.GetConfigureFile( build_dir ):
print ("Error in GetStamp function, could not find Configure.xml")
return
configure_file = open( self.configure_path, "r" )
content = configure_file.read()
configure_file.close()
stamp_regex = re.compile( "\\bBuildStamp\\b=\"([0-9,\\s,\(,\),\-,\.,_,A-Z,a-z]+)\"")
stamp = stamp_regex.search( content )
if trace:
print( stamp_regex )
print( stamp )
if stamp:
if trace:
print("Stamp value \n" , stamp.group(1))
self.stamp = stamp.group(1)
return self.stamp
print("Could not retreive stamp value")
return
def GetBuildId (self, **kwargs):
"""
This function is returning the buildid. Dict can be passed with the
different informations
"""
site = self.site
stamp = self.stamp
name = self.name
project = self.project
for key , value in kwargs.items():
if key == "site":
site = value
if key == "stamp":
stamp = value
if key == "name":
name = value
if key == "project":
project = value
if ( site == "" or stamp == "" or name == "" or project == ""):
print( "Not enougth argument given for buildid request \
site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
buildid_api = "/api/v1/getbuildid.php?"
buildid_params = urllib.parse.urlencode({'project': project, 'site': site, 'stamp': stamp , 'name': name})
full_url = self.url + buildid_api + buildid_params
if trace:
print("full_url: "+full_url)
response = urllib.request.urlopen(full_url).read().decode()
if trace:
print ( "response: " + response )
build_id_regex = re.compile( "<buildid>([0-9]+)</buildid>" )
buildid = build_id_regex.search( response )
if buildid:
self.buildid = buildid.group(1)
if trace:
print ( "build id is ", self.buildid)
return buildid.group(1)
else:
print("Error in recovering buildid")
return
def GetBuildUrl (self , buildid = "" ):
"""
This function is returning the build url. It can be called only when
everything is set
"""
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Not enougth argument given to build url")
return
build_url = self.url
build_url +="/buildSummary.php?"
build_url += "buildid=" + buildid
return build_url
def GetBuildStatus(self, buildid = "" ):
"""
This function returns the status of a build id as a pair 'state' + 'errors'
"""
if ( buildid == "" ):
buildid = self.buildid
if ( buildid == "" ):
print( "Not enougth argument given to build Status")
return
full_url = self.url + "/api/v1/buildSummary.php?buildid=" + buildid
response = urllib.request.urlopen(full_url).read().decode()
full_status = json.loads(response)
state = "success"
if full_status["configure"]["nerrors"] or full_status["build"]["nerrors"] \
or full_status["test"]["nerrors"] or full_status["test"]["nfailed"]:
state = 'failed'
errors = ""
if full_status["configure"]["nerrors"]:
errors = "Errors occur during configure"
elif full_status["build"]["nerrors"]:
errors = "Errors occur during build"
elif full_status["test"]["nerrors"] or full_status["test"]["nfailed"]:
errors = "Errors occur during tests"
return ( state , errors)
"""
TODO :
documentation, header, test if it is possible.
the script aims only at recovering the build url
"""
if __name__ == "__main__":
if ( len(sys.argv) < 5 ):
print("Usage : "+sys.argv[0]+" commit_sha1 project_id build_directory token")
sys.exit()
if trace:
print (sys.argv)
handler = Handler()
build_dir = os.path.join( sys.argv[3] , "build/")
# build_dir = sys.argv[3]
if trace:
print("build_dir is: " + build_dir)
handler.build_dir = build_dir
handler.GetSite()
handler.GetName()
handler.GetStamp()
handler.GetBuildId()
# handler.buildid="1"
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetBuildStatus()
if trace:
print ( "cdash_url is: " + cdash_url )
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url += sys.argv[2] + "/statuses/" + sys.argv[1]
params = urllib.parse.urlencode({'name':'cdash:' + handler.site , 'state': state ,\
'target_url' : cdash_url , 'description' : error })
gitlab_request = urllib.request.Request(gitlab_url)
gitlab_request.add_header('PRIVATE-TOKEN' , sys.argv[4] )
res = urllib.request.urlopen(gitlab_request, data=params.encode('ascii'))
if trace:
print ("gitlab_request.url: " + gitlab_request.full_url)
print ("gitlab_request.text: " + res.read().decode())
......@@ -51,12 +51,12 @@ OTB_USE_QWT:BOOL=ON
OTB_USE_SHARK:BOOL=ON
OTB_USE_SIFTFAST:BOOL=ON
OTB_USE_SPTW:BOOL=ON
OTB_USE_SSE_FLAGS:BOOL=ON")
OTB_USE_SSE_FLAGS:BOOL=ON
OTB_MPIEXEC_OPT:STRING=--allow-run-as-root")
set (otb_wrap_option
"OTB_WRAP_JAVA:BOOL=ON
OTB_WRAP_PYTHON:BOOL=ON
OTB_WRAP_PYTHON3:BOOL=ON")
OTB_WRAP_PYTHON:BOOL=ON")
set (otb_data_option
"OTB_DATA_USE_LARGEINPUT:BOOL=OFF
......
......@@ -45,6 +45,8 @@ elseif(ci_ref_name)
endif()
endif()
#Warning, this variable is used in cdash_status.py. If change from
# ${IMAGE_NAME} to something else do not forget to change it.
set (CTEST_SITE "${IMAGE_NAME}")
# Directory variable
......@@ -86,6 +88,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
)
if ( NOT _configure_rv EQUAL 0 )
# stop processing here
ctest_submit()
message( FATAL_ERROR "An error occurs during ctest_configure.")
endif()
......@@ -96,19 +99,16 @@ ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
)
if ( NOT _build_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_build.")
endif()
# Uncomment when ready for test
# ctest_test(PARALLEL_LEVEL 8
# RETURN_VALUE _test_rv
# CAPTURE_CMAKE_ERROR _test_error
# )
ctest_test(PARALLEL_LEVEL 8
RETURN_VALUE _test_rv
CAPTURE_CMAKE_ERROR _test_error
)
# if ( NOT _test_rv EQUAL 0 )
# ctest_submit()
# message( SEND_ERROR "An error occurs during ctest_test.")
# endif()
if ( NOT _test_rv EQUAL 0 )
message( SEND_ERROR "An error occurs during ctest_test.")
endif()
ctest_submit()
This folder aims at simulating a build directory. It contains the sub-durectory Testing with a Configure.xml file as the one generated by ctest.
<?xml version="1.0" encoding="UTF-8"?>
<Site BuildName="185_f-54sf_ghs-dgh (out_space)"
BuildStamp="20190320-1706-Experimental_test"
Name="debian-unstable-gcc.42"
Generator="ctest-3.13.4"
CompilerName=""
CompilerVersion=""
OSName="Linux"
Hostname="0e82a8db0490"
OSRelease="4.13.0-43-generic"
OSVersion="#48~16.04.1-Ubuntu SMP Thu May 17 12:56:46 UTC 2018"
OSPlatform="x86_64"
Is64Bits="1"
VendorString="GenuineIntel"
VendorID="Intel Corporation"
FamilyID="6"
ModelID="78"
ProcessorCacheSize="3072"
NumberOfLogicalCPU="4"
NumberOfPhysicalCPU="2"
TotalVirtualMemory="5720"
TotalPhysicalMemory="7858"
LogicalProcessorsPerPhysical="2"
ProcessorClockFrequency="1626.81"
>
<Configure>
<StartDateTime>Mar 20 17:06 UTC</StartDateTime>
<StartConfigureTime>1553101581</StartConfigureTime>
<ConfigureCommand>--</ConfigureCommand>
<Log>--</Log>
<ConfigureStatus>0</ConfigureStatus>
<EndDateTime>Mar 20 17:06 UTC</EndDateTime>
<EndConfigureTime>1553101601</EndConfigureTime>
<ElapsedMinutes>0</ElapsedMinutes>
</Configure>
</Site>
import unittest
import os.path
import sys
# This is done so we can access the module handler
synthetic_build_dir = os.path.dirname(os.path.realpath(__file__))
(module_path , _) = os.path.split(synthetic_build_dir)
sys.path.append(module_path)
import cdash_handler
class TestHandler(unittest.TestCase):
def test_GetConfigureFile (self):
handler = cdash_handler.Handler()
handler.build_dir = synthetic_build_dir
config_path = handler.GetConfigureFile()
self.assertTrue(os.path.exists(config_path))
def test_GetName (self):
handler = cdash_handler.Handler()
handler.build_dir = synthetic_build_dir
name = handler.GetName()
self.assertTrue("185_f-54sf_ghs-dgh (out_space)" == name)
def test_GetSite (self):
handler = cdash_handler.Handler()
handler.build_dir = synthetic_build_dir
site = handler.GetSite()
self.assertTrue("debian-unstable-gcc.42" == site)
def test_GetStamp (self):
handler = cdash_handler.Handler()
handler.build_dir = synthetic_build_dir
stamp = handler.GetStamp()
self.assertTrue("20190320-1706-Experimental_test" == stamp)
def test_GetBuildId (self):
handler = cdash_handler.Handler()
buildid= handler.GetBuildId( site="debian-unstable-gcc" ,\
stamp="20190319-1243-Experimental" , name="0388a356 (debian_CI)")
self.assertTrue( "1" == buildid )
def test_GetBuildUrl (self):
handler = cdash_handler.Handler()
url = handler.GetBuildUrl("1")
self.assertTrue( \
"https://cdash.orfeo-toolbox.org/buildSummary.php?buildid=1" == url)
def test_GetBuildStatus (self):
handler = cdash_handler.Handler()
# We need to test error in configure, build, test, and no error
# test buildid 720
(status , error) = handler.GetBuildStatus("720")
self.assertTrue( status == "failed")
self.assertTrue( error == "Errors occur during tests")
# configure buildid 352
(status , error) = handler.GetBuildStatus("352")
self.assertTrue( status == "failed")
self.assertTrue( error == "Errors occur during configure")
# build buildid 358
(status , error) = handler.GetBuildStatus("358")
self.assertTrue( status == "failed")
self.assertTrue( error == "Errors occur during build")
# passing buildid 373
(status , error) = handler.GetBuildStatus("373")
self.assertTrue( status == "success")
self.assertTrue( error == "")
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
......@@ -27,5 +27,5 @@ CMAKE_CXX_COMPILER:STRING=clang++
CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache")
OTB_USE_SHARK:BOOL=OFF
")
......@@ -224,7 +224,7 @@ endfunction()
function(otb_add_test_mpi)
set( _OPTIONS_ARGS )
set( _ONE_VALUE_ARGS NAME NBPROCS COMMAND)
set( _MULTI_VALUE_ARGS )
set( _MULTI_VALUE_ARGS EXTRA_OPT)
cmake_parse_arguments( TEST_MPI "${_OPTIONS_ARGS}" "${_ONE_VALUE_ARGS}" "${_MULTI_VALUE_ARGS}" ${ARGN} )
# Test nb procs
......@@ -235,6 +235,6 @@ function(otb_add_test_mpi)
foreach(arg IN LISTS TEST_MPI_UNPARSED_ARGUMENTS)
list(APPEND ARGS ${arg})
endforeach()
set (test_parameters -np ${TEST_MPI_NBPROCS} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_MPI_COMMAND} ${ARGS})
set (test_parameters -n ${TEST_MPI_NBPROCS} ${OTB_MPIEXEC_OPT} ${TEST_MPI_EXTRA_OPT} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_MPI_COMMAND} ${ARGS})
otb_add_test(NAME ${TEST_MPI_NAME} COMMAND ${MPIEXEC} ${test_parameters})
endfunction()
......@@ -101,7 +101,7 @@ endmacro()
function(otb_add_test_mpi)
set( _OPTIONS_ARGS )
set( _ONE_VALUE_ARGS NAME NBPROCS COMMAND)
set( _MULTI_VALUE_ARGS )
set( _MULTI_VALUE_ARGS EXTRA_OPT)
cmake_parse_arguments( TEST_MPI "${_OPTIONS_ARGS}" "${_ONE_VALUE_ARGS}" "${_MULTI_VALUE_ARGS}" ${ARGN} )
# Test nb procs
......@@ -112,7 +112,7 @@ function(otb_add_test_mpi)
foreach(arg IN LISTS TEST_MPI_UNPARSED_ARGUMENTS)
list(APPEND ARGS ${arg})
endforeach()
set (test_parameters -np ${TEST_MPI_NBPROCS} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_MPI_COMMAND} ${ARGS})
set (test_parameters -n ${TEST_MPI_NBPROCS} ${OTB_MPIEXEC_OPT} ${TEST_MPI_EXTRA_OPT} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_MPI_COMMAND} ${ARGS})
otb_add_test(NAME ${TEST_MPI_NAME} COMMAND ${MPIEXEC} ${test_parameters})
endfunction()
......
%YAML:1.0
---
opencv_ml_dtree:
format: 3
is_classifier: 1
var_all: 5
var_count: 4
ord_var_count: 4
cat_var_count: 1
training_params:
use_surrogates: 0
max_categories: 10
regression_accuracy: 9.9999997764825821e-03
max_depth: 10
min_sample_count: 10
cross_validation_folds: 0
global_var_idx: 1
var_idx: [ 0, 1, 2, 3 ]
var_type: [ 0, 0, 0, 0, 1 ]
cat_ofs: [ 0, 0, 0, 0, 0, 0, 0, 0 ]
class_labels: [ 1, 2, 3, 4 ]
missing_subst: [ 0., 0., 0., 0., 0. ]
nodes:
-
depth: 0
value: 1.
norm_class_idx: 0
splits:
- { var:0, quality:86., le:-8.9000031352043152e-02 }
-
depth: 1
value: 1.
norm_class_idx: 0
splits:
- { var:3, quality:86., le:-1.3876452445983887e+00 }
-
depth: 2
value: 1.
norm_class_idx: 0
-
depth: 2
value: 3.
norm_class_idx: 2
-
depth: 1
value: 2.
norm_class_idx: 1
splits:
- { var:1, quality:86., le:3.4949243068695068e-01 }
-
depth: 2
value: 2.
norm_class_idx: 1
-
depth: 2
value: 4.
norm_class_idx: 3
......@@ -19,7 +19,7 @@
*/
#include "itkMacro.h"
#include "otbMacro.h"
#include "otbImage.h"
#include "otbImageFileReader.h"
......@@ -166,7 +166,7 @@ int main(int argc, char* argv[])
{
typedef OutputPointSetType::PointType OutputPointType;
OutputPointType pCoordinate = (itList.Value());
std::cout << pCoordinate << std::endl;
otbLogMacro(Debug, << pCoordinate);
++itList;
}
......