Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Main Repositories
otb
Commits
61f55ac1
Commit
61f55ac1
authored
Jun 27, 2019
by
Guillaume Pasero
Browse files
Merge branch 'ci_alpha_fixes' into 'develop'
CI Fixes episode N See merge request
orfeotoolbox/otb!536
parents
4facfd77
167c91e6
Pipeline
#2035
passed with stages
in 6 minutes and 44 seconds
Changes
4
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
61f55ac1
...
...
@@ -34,26 +34,11 @@ stages:
.common
:
extends
:
.general
before_script
:
-
git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
-
git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
-
git checkout -f -q $CI_COMMIT_SHA
after_script
:
-
python3 CI/cdash_handler.py
fast-build
:
extends
:
.common
only
:
[
merge_requests
,
branches
]
stage
:
precheck
image
:
$BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script
:
-
export GIT_LFS_SKIP_SMUDGE=1
-
git checkout -f -q $CI_COMMIT_SHA
-
python3 CI/check_twin_pipelines.py
script
:
-
ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
-
ccache -s
-
python3 -u CI/cdash_handler.py
.common-build
:
extends
:
.common
...
...
@@ -74,12 +59,6 @@ fast-build:
-
build_packages/OTB-*.run
-
build_packages/OTB-*.zip
debian-build
:
extends
:
.common-build
image
:
$BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script
:
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
.common-prepare
:
extends
:
.general
only
:
...
...
@@ -88,7 +67,6 @@ debian-build:
-
/^release-[0-9]+\.[0-9]+$/
stage
:
prepare
before_script
:
-
git lfs install --skip-repo
-
export GIT_LFS_SKIP_SMUDGE=1
-
git checkout -f -q $CI_COMMIT_SHA
-
export GIT_LFS_SKIP_SMUDGE=0
...
...
@@ -99,6 +77,27 @@ debian-build:
-
sb_branch.txt
# Needed to checkout correct branch in build step
-
build/*/*/*/*.log
# Superbuild log
#-------------------------- precheck job ---------------------------------------
fast-build
:
extends
:
.common
only
:
[
merge_requests
,
branches
]
stage
:
precheck
image
:
$BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script
:
-
export GIT_LFS_SKIP_SMUDGE=1
-
git checkout -f -q $CI_COMMIT_SHA
-
python3 CI/check_twin_pipelines.py
script
:
-
ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-fast
-
ccache -s
#------------------------- prepare & build jobs --------------------------------
debian-build
:
extends
:
.common-build
image
:
$BUILD_IMAGE_REGISTRY/otb-debian-native:unstable
script
:
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_ci.cmake -DIMAGE_NAME:string=debian-unstable-gcc
## Ubuntu superbuild
ubuntu-xdk-prepare
:
extends
:
.common-prepare
...
...
@@ -111,7 +110,7 @@ ubuntu-xdk-build:
image
:
$BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script
:
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V
V
-S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
dependencies
:
-
ubuntu-xdk-prepare
...
...
@@ -127,20 +126,15 @@ centos-xdk-build:
image
:
$BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script
:
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V
V
-S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
-
xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies
:
-
centos-xdk-prepare
## MacOS superbuild
macos-xdk-prepare
:
extends
:
.common-prepare
tags
:
-
macos
extends
:
.common-prepare
before_script
:
# No need to install lfs as this machine is persistent
-
export GIT_LFS_SKIP_SMUDGE=1
-
git checkout -f -q $CI_COMMIT_SHA
-
export GIT_LFS_SKIP_SMUDGE=0
script
:
-
ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
...
...
@@ -150,7 +144,7 @@ macos-xdk-build:
extends
:
.common-build
script
:
-
ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
-
ctest -V
V
-S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
-
ctest -V -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies
:
-
macos-xdk-prepare
...
...
@@ -217,9 +211,7 @@ windows-8-build:
dependencies
:
-
windows-8-prepare
# Deploy job
#---------------------------- Deploy job ---------------------------------------
deploy
:
tags
:
-
deploy
...
...
@@ -227,13 +219,11 @@ deploy:
stage
:
deploy
extends
:
.general
only
:
-
develop
-
/^release-[0-9]+\.[0-9]+$/
-
develop
@orfeotoolbox/otb
-
/^release-[0-9]+\.[0-9]+$/
@orfeotoolbox/otb
before_script
:
-
git lfs install --skip-repo
# Provision efficiently the local LFS cache before checkout
-
git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
-
git checkout -f -q $CI_COMMIT_SHA
script
:
-
./CI/deploy.sh $CI_COMMIT_REF_NAME $RC_NUMBER
...
...
@@ -242,4 +232,16 @@ deploy:
-
centos-xdk-build
-
macos-xdk-build
-
windows-8-build
-
windows-10-build
\ No newline at end of file
-
windows-10-build
release-container
:
image
:
$BUILD_IMAGE_REGISTRY/otb-alpine:3.7
stage
:
deploy
extends
:
.general
only
:
refs
:
-
tags@orfeotoolbox/otb
variables
:
-
$CI_COMMIT_TAG =~ /^[0-9]+\.[0-9]+\.[0-9]+$/
script
:
-
"
curl
--request
POST
--form
token=$K8S_SECRET_RELEASE
--form
ref=master
--form
variables[OTB_TAG]=$CI_COMMIT_TAG
https://gitlab.orfeo-toolbox.org/api/v4/projects/126/trigger/pipeline"
CI/cdash_handler.py
View file @
61f55ac1
...
...
@@ -28,6 +28,7 @@ import unittest
import
sys
import
json
import
time
import
xml.etree.ElementTree
as
ET
trace
=
False
...
...
@@ -42,11 +43,10 @@ def CheckEnvParameters(params):
return
False
return
True
"""
Handler class to retrieve build informations
"""
class
Handler
:
# project
# site
# stamp
# name
def
__init__
(
self
):
self
.
build_dir
=
""
self
.
configure_path
=
""
...
...
@@ -75,91 +75,39 @@ class Handler:
if
os
.
path
.
exists
(
configure_xml
):
if
trace
:
print
(
configure_xml
)
configure_file
=
open
(
configure_xml
,
"r"
)
content
=
configure_file
.
read
()
configure_file
.
close
()
print
(
content
)
self
.
configure_path
=
configure_xml
return
self
.
configure_path
print
(
"Could not find the Configure.xml produced by ctest"
)
return
def
GetSite
(
self
,
build_dir
=
""
):
"""
Site is corresponding to the Name field in the xml.
"""
if
(
build_dir
==
""
):
build_dir
=
self
.
build_dir
if
self
.
configure_path
==
""
and
not
self
.
GetConfigureFile
(
build_dir
):
print
(
"Error in GetSite function, could not find Configure.xml"
)
return
configure_file
=
open
(
self
.
configure_path
,
"r"
)
content
=
configure_file
.
read
()
configure_file
.
close
()
site_regex
=
re
.
compile
(
"
\\
bName
\\
b=
\"
([0-9,
\\
s,\(,\),\-,\.,_,A-Z,a-z]+)"
)
site
=
site_regex
.
search
(
content
)
if
trace
:
print
(
site_regex
)
print
(
site
)
if
site
:
if
trace
:
print
(
"site value
\n
"
,
site
.
group
(
1
))
self
.
site
=
site
.
group
(
1
)
return
self
.
site
print
(
"Could not retreive site value"
)
return
return
def
GetName
(
self
,
build_dir
=
""
):
"""
This function is looking for the name information in the build tree:
which is BuildName
"""
if
(
build_dir
==
""
):
build_dir
=
self
.
build_dir
if
self
.
configure_path
==
""
and
not
self
.
GetConfigureFile
(
build_dir
):
print
(
"Error in GetName function, could not find Configure.xml"
)
return
configure_file
=
open
(
self
.
configure_path
,
"r"
)
content
=
configure_file
.
read
()
configure_file
.
close
()
name_regex
=
re
.
compile
(
"
\\
bBuildName
\\
b=
\"
([0-9,
\\
s,\(,\),\-,\.,_,A-Z,a-z]+)
\"
"
)
name
=
name_regex
.
search
(
content
)
if
trace
:
print
(
name_regex
)
print
(
name
)
if
name
:
if
trace
:
print
(
"name value
\n
"
,
name
.
group
(
1
))
self
.
name
=
name
.
group
(
1
)
return
self
.
name
print
(
"Could not retreive name value"
)
return
sys
.
exit
(
1
)
def
GetStamp
(
self
,
build_dir
=
""
):
def
ParseConfigureFile
(
self
):
"""
This function is looking for the stamp information in the build tree
Parse the configuration file to get Name, Site and BuildStamp
"""
if
(
build_dir
==
""
):
build_dir
=
self
.
build_dir
if
self
.
configure_path
==
""
and
not
self
.
GetConfigureFile
(
build_dir
):
print
(
"Error in GetStamp function, could not find Configure.xml"
)
return
configure_file
=
open
(
self
.
configure_path
,
"r"
)
content
=
configure_file
.
read
()
configure_file
.
close
()
stamp_regex
=
re
.
compile
(
"
\\
bBuildStamp
\\
b=
\"
([0-9,
\\
s,\(,\),\-,\.,_,A-Z,a-z]+)
\"
"
)
stamp
=
stamp_regex
.
search
(
content
)
# strip the Log section as it can mess up the XML parser
startLog
=
content
.
find
(
'<Log>'
)
endLog
=
content
.
rfind
(
'</Log>'
)
if
startLog
>
0
and
endLog
>
startLog
:
content
=
content
[:(
startLog
+
5
)]
+
content
[
endLog
:]
# parse XML
root
=
ET
.
fromstring
(
content
)
if
trace
:
print
(
stamp_regex
)
print
(
stamp
)
if
stamp
:
if
trace
:
print
(
"Stamp value
\n
"
,
stamp
.
group
(
1
))
self
.
stamp
=
stamp
.
group
(
1
)
return
self
.
stamp
print
(
"Could not retreive stamp value"
)
return
print
(
root
.
attrib
)
if
not
'Name'
in
root
.
keys
():
print
(
"Can't find site name in Configure.XML"
)
sys
.
exit
(
1
)
if
not
'BuildName'
in
root
.
keys
():
print
(
"Can't find build name in Configure.XML"
)
sys
.
exit
(
1
)
if
not
'BuildStamp'
in
root
.
keys
():
print
(
"Can't find build stamp in Configure.XML"
)
sys
.
exit
(
1
)
self
.
site
=
root
.
get
(
'Name'
)
self
.
name
=
root
.
get
(
'BuildName'
)
self
.
stamp
=
root
.
get
(
'BuildStamp'
)
def
GetBuildId
(
self
,
**
kwargs
):
"""
...
...
@@ -180,15 +128,17 @@ class Handler:
if
key
==
"project"
:
project
=
value
if
(
site
==
""
or
stamp
==
""
or
name
==
""
or
project
==
""
):
print
(
"Missing argument for buildid request
\
site:"
+
site
+
", stamp:"
+
stamp
+
", name:"
+
name
+
", project:"
+
project
+
"."
)
return
print
(
"Missing argument for buildid request site:"
+
site
+
", stamp:"
+
stamp
+
", name:"
+
name
+
", project:"
+
project
+
"."
)
sys
.
exit
(
1
)
elif
trace
:
print
(
"Argument for buildid request site:"
+
site
+
", stamp:"
+
stamp
+
", name:"
+
name
+
", project:"
+
project
+
"."
)
buildid_api
=
"/api/v1/getbuildid.php?"
buildid_params
=
urllib
.
parse
.
urlencode
({
'project'
:
project
,
'site'
:
site
,
'stamp'
:
stamp
,
'name'
:
name
})
full_url
=
self
.
url
+
buildid_api
+
buildid_params
if
trace
:
print
(
"full_url: "
+
full_url
)
nb_try
=
6
max_retry
=
11
nb_try
=
max_retry
build_id_regex
=
re
.
compile
(
"<buildid>([0-9]+)</buildid>"
)
while
nb_try
:
response
=
urllib
.
request
.
urlopen
(
full_url
).
read
().
decode
()
...
...
@@ -198,8 +148,8 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
nb_try
-=
1
if
buildid
or
(
nb_try
==
0
):
break
print
(
"No build id, retry ..."
)
time
.
sleep
(
6
0
)
print
(
"No build id, retry
"
+
str
(
max_retry
-
nb_try
)
+
"/"
+
str
(
max_retry
)
+
"
..."
)
time
.
sleep
(
3
0
)
if
buildid
:
self
.
buildid
=
buildid
.
group
(
1
)
if
trace
:
...
...
@@ -207,7 +157,7 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
return
buildid
.
group
(
1
)
else
:
print
(
"Error in recovering buildid"
)
return
sys
.
exit
(
1
)
def
GetBuildUrl
(
self
,
buildid
=
""
):
"""
...
...
@@ -290,11 +240,11 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
cdash_handler.py commit_sha1 project_id project_directory token ref_name
"""
if
__name__
==
"__main__"
:
if
trace
:
print
(
sys
.
argv
)
if
(
len
(
sys
.
argv
)
<
6
and
len
(
sys
.
argv
)
>
1
):
print
(
"Usage : "
+
sys
.
argv
[
0
]
+
" commit_sha1 project_id project_directory token ref_name"
)
sys
.
exit
(
1
)
if
trace
:
print
(
sys
.
argv
)
if
(
len
(
sys
.
argv
)
>=
6
):
sha1
=
sys
.
argv
[
1
]
proj
=
sys
.
argv
[
2
]
...
...
@@ -302,24 +252,26 @@ if __name__ == "__main__":
token
=
sys
.
argv
[
4
]
refn
=
sys
.
argv
[
5
]
else
:
if
not
CheckEnvParameters
([
'CI_COMMIT_SHA'
,
'CI_PROJECT_ID'
,
'CI_PROJECT_DIR'
,
'K8S_SECRET_API_TOKEN'
,
'CI_COMMIT_REF_NAME'
]):
if
not
CheckEnvParameters
([
'CI_COMMIT_SHA'
,
'CI_PROJECT_ID'
,
'CI_PROJECT_DIR'
,
'CI_COMMIT_REF_NAME'
]):
sys
.
exit
(
1
)
sha1
=
os
.
environ
[
'CI_COMMIT_SHA'
]
proj
=
os
.
environ
[
'CI_PROJECT_ID'
]
pdir
=
os
.
environ
[
'CI_PROJECT_DIR'
]
token
=
os
.
environ
[
'K8S_SECRET_API_TOKEN'
]
if
'CI_MERGE_REQUEST_REF_PATH'
in
os
.
environ
.
keys
():
refn
=
os
.
environ
[
'CI_MERGE_REQUEST_REF_PATH'
]
else
:
refn
=
os
.
environ
[
'CI_COMMIT_REF_NAME'
]
if
CheckEnvParameters
([
'K8S_SECRET_API_TOKEN'
]):
token
=
os
.
environ
[
'K8S_SECRET_API_TOKEN'
]
else
:
token
=
None
handler
=
Handler
()
build_dir
=
os
.
path
.
join
(
pdir
,
"build/"
)
if
trace
:
print
(
"build_dir is: "
+
build_dir
)
handler
.
build_dir
=
build_dir
handler
.
GetSite
()
handler
.
GetName
()
handler
.
GetStamp
()
handler
.
GetConfigureFile
()
handler
.
ParseConfigureFile
()
if
handler
.
GetBuildId
()
is
None
:
cdash_url
=
"https://cdash.orfeo-toolbox.org"
state
=
'failed'
...
...
@@ -327,8 +279,9 @@ if __name__ == "__main__":
else
:
cdash_url
=
handler
.
GetBuildUrl
()
(
state
,
error
)
=
handler
.
GetLogStatus
(
os
.
path
.
join
(
pdir
,
"log"
)
)
if
trace
:
print
(
"cdash_url is: "
+
cdash_url
)
print
(
"CDash build URL : "
+
cdash_url
)
if
token
is
None
:
sys
.
exit
(
0
)
gitlab_url
=
"https://gitlab.orfeo-toolbox.org/api/v4/projects/"
gitlab_url
+=
proj
+
"/statuses/"
+
sha1
params
=
urllib
.
parse
.
urlencode
({
'name'
:
'cdash:'
+
handler
.
site
,
'state'
:
state
,
\
...
...
CI/check_twin_pipelines.py
View file @
61f55ac1
...
...
@@ -24,6 +24,7 @@ import urllib.parse
import
json
import
re
import
time
import
sys
"""
Send a request to Gitlab and return the answer
...
...
@@ -65,9 +66,10 @@ if __name__ == "__main__":
# are we in a merge_request pipeline ?
if
'CI_MERGE_REQUEST_IID'
in
env
.
keys
():
if
not
CheckEnvParameters
([
'K8S_SECRET_API_TOKEN'
]):
print
(
"Make sure you have set a valid acces token for Gitlab API"
)
print
(
"The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'"
)
sys
.
exit
(
1
)
print
(
"WARNING: Make sure you have set a valid acces token for Gitlab API."
\
+
"The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'"
\
+
"Without this token, some feature of the CI platform will be disabled."
)
sys
.
exit
(
0
)
if
not
CheckEnvParameters
([
'CI_PROJECT_ID'
,
'CI_PIPELINE_ID'
]):
sys
.
exit
(
1
)
mrInfo
=
GitlabRequest
(
'merge_requests/'
+
env
[
'CI_MERGE_REQUEST_IID'
],
token
=
env
[
'K8S_SECRET_API_TOKEN'
])
...
...
Packaging/detect_using_file_command.cmake
View file @
61f55ac1
...
...
@@ -115,10 +115,10 @@ function(detect_using_file_command input_file result_type result_dir)
# Patch file command returning shared-oject for executable on Linux when -PIE is used.
if
(
LINUX
)
get_filename_component
(
input_file_DIR
${
input_file
}
DIRECTORY
)
get_filename_component
(
input_file_dIR
${
input_file_DIR
}
NAME
)
string
(
TOLOWER
"
${
input_file_DIR
}
"
input_file_DIR
)
if
(
${
input_file_DIR
}
MATCHES
"bin"
)
message
(
WARNING
"
${
input_file
}
detected as shared-object
.
"
)
get_filename_component
(
dir_name
${
input_file_DIR
}
NAME
)
string
(
TOLOWER
"
${
dir_name
}
"
dir_name_l
)
if
(
"
${
dir_name_l
}
"
STREQUAL
"bin"
)
message
(
STATUS
"
${
input_file
}
detected as shared-object
, but processed as executable
"
)
set
(
detected_type PROGRAMS
)
set
(
detected_dir bin
)
endif
()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment