Commit 64257c82 authored by Antoine Regimbeau's avatar Antoine Regimbeau
Browse files

Merge branch 'develop' into ci_deploy_script

parents e9407e68 fe2a6982
......@@ -41,12 +41,6 @@ stages:
- git checkout -f -q $CI_COMMIT_SHA
after_script:
- python3 CI/cdash_handler.py
artifacts:
when: on_failure
expire_in: 24 hrs
paths:
- build/*/*.log #CMake log
- log/*.txt # Others
fast-build:
extends: .common
......@@ -54,6 +48,7 @@ fast-build:
stage: precheck
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-native-develop:latest
before_script:
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- python3 CI/check_twin_pipelines.py
script:
......@@ -67,6 +62,17 @@ fast-build:
- develop
- /^release-[0-9]+\.[0-9]+$/
stage: build
artifacts:
when: always
expire_in: 24 hrs
paths:
- build/*/*.log #CMake log
- log/*.txt # Others
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
- build_packages/OTB-*.run
- build_packages/OTB-*.zip
debian-build:
extends: .common-build
......@@ -82,10 +88,10 @@ debian-build:
- /^release-[0-9]+\.[0-9]+$/
stage: prepare
before_script:
- git checkout -f -q $CI_COMMIT_SHA
- git lfs install --skip-repo
- git config --global user.email "otbbot@orfeo-toolbox.org"
- git config --global user.name "otbbot"
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
artifacts:
expire_in: 24 hrs
when: always
......@@ -94,62 +100,51 @@ debian-build:
- build/*/*/*/*.log # Superbuild log
## Ubuntu superbuild
ubuntu-superbuild-prepare:
ubuntu-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base
ubuntu-superbuild-build:
ubuntu-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-ubuntu-superbuild-base:18.04
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-xdk
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-ubuntu-superbuild-base -DNAME_SUFFIX:string=-glibc-2.27
dependencies:
- ubuntu-superbuild-prepare
artifacts:
paths:
- build/CookBook-*-html.tar.gz
- build/Documentation/Cookbook/latex/CookBook-*.pdf
- build/Documentation/Doxygen/OTB-Doxygen-*.tar.bz2
- build_packages/OTB-*.run
- ubuntu-xdk-prepare
## CentOS superbuild
centos-superbuild-prepare:
centos-xdk-prepare:
extends: .common-prepare
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
centos-superbuild-build:
centos-xdk-build:
extends: .common-build
image: $BUILD_IMAGE_REGISTRY/otb-centos-superbuild-base:6.6
script:
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
- xvfb-run -a -n 1 -s "-screen 0 1024x768x24 -dpi 96" ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-centos-superbuild-base
dependencies:
- centos-superbuild-prepare
artifacts:
paths:
- build_packages/OTB-*.run
- centos-xdk-prepare
## MacOS superbuild
macos-superbuild-prepare:
macos-xdk-prepare:
tags:
- macos
extends: .common-prepare
before_script:
# No need to install lfs as this machine is persistent
# No need to do git config
# Checkout the expected branch
- export GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q $CI_COMMIT_SHA
- export GIT_LFS_SKIP_SMUDGE=0
script:
- ctest -VV -S CI/prepare_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
macos-superbuild-build:
macos-xdk-build:
tags:
- macos
extends: .common-build
......@@ -157,11 +152,72 @@ macos-superbuild-build:
- ctest -V -S CI/main_superbuild.cmake -DIMAGE_NAME:string=otb-macos-superbuild
- ctest -VV -S CI/main_packages.cmake -DIMAGE_NAME:string=otb-macos-superbuild
dependencies:
- macos-superbuild-prepare
artifacts:
paths:
- build_packages/OTB-*.run
- macos-xdk-prepare
## Windows
.windows-prepare:
extends: .common-prepare
before_script:
# This override the previous before_script
- set GIT_LFS_SKIP_SMUDGE=1
- git checkout -f -q %CI_COMMIT_SHA%
- set GIT_LFS_SKIP_SMUDGE=0
.windows-build:
extends: .common-build
before_script:
- git lfs fetch origin %CI_COMMIT_SHA%
- git checkout -f -q %CI_COMMIT_SHA%
# - Win10
windows-10-prepare:
extends: .windows-prepare
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 xdk 10
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-10-build:
extends: .windows-build
tags:
- windows10
script:
- call ./CI/dev_env.bat x64 otb 10
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-10-prepare
# - Win8.1
windows-8-prepare:
extends: .windows-prepare
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 xdk 8.1
- clcache.exe -s
- ctest -C Release -V -S CI/prepare_superbuild.cmake
- clcache.exe -s
windows-8-build:
extends: .windows-build
tags:
- windows8
script:
- call ./CI/dev_env.bat x86 otb 8.1
- clcache.exe -s
- ctest -V -S CI/main_superbuild.cmake
- clcache.exe -s
- ctest -V -S CI/main_packages.cmake
dependencies:
- windows-8-prepare
# Deploy job
deploy:
......
......@@ -30,7 +30,7 @@ import json
import time
trace = False
trace = True
"""
Check needed environment parameters
......@@ -249,6 +249,34 @@ site:"+site+", stamp:"+stamp+", name:"+name+", project:"+project+".")
errors = "Errors occur during tests"
return ( state , errors)
def GetReturnValue(self, logfile):
fd = open(logfile)
content = fd.readlines()[0]
fd.close()
return int(content.strip("\n"))
def GetLogStatus(self, logdir):
"""
This function returns the log status of a build as a pair 'state' + 'errors'
"""
configure_rv = os.path.join(logdir, "configure_return_value_log.txt")
build_rv = os.path.join(logdir, "build_return_value_log.txt")
test_rv = os.path.join(logdir, "test_return_value_log.txt")
if os.path.exists( configure_rv ):
if (self.GetReturnValue(configure_rv) != 0):
return ( 'failed' , 'Configure failed')
else:
return ( 'failed' , 'Configure not run')
if os.path.exists( build_rv ):
if (self.GetReturnValue(build_rv) != 0):
return ( 'failed' , 'Build failed')
else:
return ( 'failed' , 'Build not run')
if os.path.exists( test_rv ):
if (self.GetReturnValue(test_rv) != 0):
return ( 'failed' , 'Tests failed')
return ('success', '')
"""
This script aims only at recovering the build url
It uses environment variables setup by Gitlab Runner as default:
......@@ -298,7 +326,7 @@ if __name__ == "__main__":
error = "Failed to get build id"
else:
cdash_url = handler.GetBuildUrl()
( state , error ) = handler.GetBuildStatus()
( state , error ) = handler.GetLogStatus( os.path.join( pdir , "log") )
if trace:
print ( "cdash_url is: " + cdash_url )
gitlab_url = "https://gitlab.orfeo-toolbox.org/api/v4/projects/"
......
......@@ -58,10 +58,6 @@ OTB_USE_SSE_FLAGS:BOOL=ON")
set (otb_wrap_option
"OTB_WRAP_PYTHON:BOOL=ON")
set (otb_data_option
"OTB_DATA_USE_LARGEINPUT:BOOL=OFF
OTB_DATA_LARGEINPUT_ROOT:PATH=${OTB_LARGEINPUT_ROOT}")
set (cmake_configure_option
"CMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION}
CMAKE_INSTALL_PREFIX:PATH=${CTEST_INSTALL_DIRECTORY}")
......
@echo off
:: check input arguments
if %1.==. (
echo "No arch"
call :Help
goto :eof
)
if %2.==. (
echo "No project"
call :Help
goto :eof
)
if /I "%1"=="help" (
call :Help
goto :eof
)
if /I "%1"=="/help" (
call :Help
goto :eof
)
if /I "%1"=="-help" (
call :Help
goto :eof
)
set ARCH=%1
set PROJECT=%2
if %3.==. (
set SHORT_TARGET=10
) else (
set SHORT_TARGET=%3
)
if %4.==. (
set VCVER=14.0
) else (
set VCVER=%4
)
set TARGET=%SHORT_TARGET%
if "%TARGET%"=="10" (
set TARGET=10.0.17763.0
)
:: Setup home dir (so that ssh configuration works fine)
:: if "%USERNAME%"=="otbbot" (
set HOMEDRIVE=C:
set HOMEPATH=\Users\otbbot
::)
echo Home dir: %HOMEDRIVE%%HOMEPATH%
:: Setup Python
set PATH=C:\tools\Python35-%ARCH%;%PATH%
set PATH=C:\tools\Python35-%ARCH%\Scripts;%PATH%
:: Setup GL dlls
set PATH=%PATH%;C:\tools\GL\%ARCH%\bin
:: Setup compiler
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" %ARCH% %TARGET% -vcvars_ver=%VCVER%
:: Setup Clcache
set CLCACHE_DIR=C:\clcache\%PROJECT%-%ARCH%-%TARGET%-%VCVER%
set CLCACHE_HARDLINK=1
:: set CLCACHE_SERVER=1
set CLCACHE_CL=
for /F "delims=" %%a in ('where cl.exe') do @if defined CLCACHE_CL (break ) else (set CLCACHE_CL=%%a)
echo CL path: "%CLCACHE_CL%"
:: install clcache.exe as cl.exe
copy C:\tools\Python35-%ARCH%\Scripts\clcache.exe C:\clcache\cl.exe
set PATH=C:\clcache;%PATH%
:: we need to change cache max size: clcache -M <size-in-bytes>
if "%PROJECT%"=="xdk" (
call "clcache.exe" -M 3000000000
)
if "%PROJECT%"=="otb" (
call "clcache.exe" -M 2000000000
)
set IMAGE_NAME=windows-%SHORT_TARGET%-%ARCH%-vc%VCVER%
echo Generated IMAGE_NAME: %IMAGE_NAME%
:: setup path to perl, but add it last ... (there is a libstdc++.dll in that folder...)
set PATH=%PATH%;C:\tools\perl\perl\bin
goto :eof
:Help
setlocal
echo "Usage: dev_env.bat <compiler_arch> <project> [<target-os> [<vc_version>]]"
echo " <compiler_arch> : 'x86' | 'x64'"
echo " <project> : 'xdk' | 'otb'"
echo " <target-os> : '8.1' | '10' (default)"
echo " <vc_version> :"
echo " '14.20' (i.e. VS 2019)"
echo " '14.16' (i.e. VS 2017)"
echo " '14.0' (i.e. VS 2015) (default)"
endlocal
......@@ -47,7 +47,7 @@ set_dash_build_name()
# set pipelines to enable documentation
set(ci_cookbook_profiles mr develop release)
set(ci_doxygen_profiles mr develop release)
set(ci_doxygen_profiles develop release)
list(FIND ci_cookbook_profiles ${ci_profile} ci_do_cookbook)
list(FIND ci_doxygen_profiles ${ci_profile} ci_do_doxygen)
......@@ -81,9 +81,6 @@ set (PROJECT_SOURCE_DIR "${OTB_SOURCE_DIR}")
# Ctest command value
set (CMAKE_COMMAND "cmake")
# Data directory setting
set (OTB_LARGEINPUT_ROOT "") # todo
message(STATUS "CI profile : ${ci_profile}")
#The following file set the CONFIGURE_OPTIONS variable
......
......@@ -29,9 +29,20 @@ get_filename_component( OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY )
set ( DEBUG "1" )
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j1" )
if(WIN32)
set ( CTEST_CMAKE_GENERATOR "NMake Makefiles JOM" )
set ( CTEST_BUILD_FLAGS "/S" )
else()
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j1")
endif()
set ( CTEST_BUILD_NAME "Packages" )
# Detect site
if(NOT DEFINED IMAGE_NAME)
if(DEFINED ENV{IMAGE_NAME})
set(IMAGE_NAME $ENV{IMAGE_NAME})
endif()
endif()
set ( CTEST_SITE "${IMAGE_NAME}" )
# Find the build name and CI profile
......@@ -63,7 +74,7 @@ set ( CONFIGURE_OPTIONS
find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
# Sources are already checked out : do nothing for update
set(CTEST_GIT_UPDATE_CUSTOM echo No update)
set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "No update")
ctest_start( Experimental TRACK CI_Package )
......
......@@ -32,17 +32,31 @@ get_xdk()
set( INSTALL_DIR "${XDK_PATH}" )
# FIX ME this part might platform dependent
set( GDAL_DATA "${XDK_PATH}/share/gdal" )
set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
set( PROJ_LIB "${XDK_PATH}/share" )
set( CTEST_ENVIRONMENT
"PATH=${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}
if(WIN32)
file(TO_NATIVE_PATH "${XDK_PATH}" XDK_PATH_NATIVE)
file(TO_NATIVE_PATH "${CTEST_BINARY_DIRECTORY}/bin" OTB_BUILD_BIN_DIR_NATIVE)
set(ENV{PATH} "$ENV{PATH};${OTB_BUILD_BIN_DIR_NATIVE}" )
set(ENV{PATH} "${XDK_PATH_NATIVE}\\bin;$ENV{PATH}" )
set(ENV{PATH} "$ENV{PATH};${XDK_PATH_NATIVE}\\lib" )
set(ENV{GDAL_DATA} "${XDK_PATH_NATIVE}\\data" )
set(ENV{GEOTIFF_CSV} "${XDK_PATH_NATIVE}\\share\\epsg_csv" )
set(ENV{PROJ_LIB} "${XDK_PATH_NATIVE}\\share" )
# needed to load Qt plugins for testing, not for binary packages where we use a qt.conf file
set(ENV{QT_PLUGIN_PATH} "${XDK_PATH_NATIVE}\\plugins")
set( CTEST_ENVIRONMENT
"PATH=$ENV{PATH}
GDAL_DATA=$ENV{GDAL_DATA}
GEOTIFF_CSV=$ENV{GEOTIFF_CSV}
PROJ_LIB=$ENV{PROJ_LIB}
")
# It seems that we do not need that
# GDAL_DATA= GDAL_DATA
# GEOTIFF_CSV= GEOTIFF_CSV
# PROJ_LIB= PROJ_LIB
else()
set(ENV{PATH} "${XDK_PATH}/lib:${XDK_PATH}/bin:$ENV{PATH}" )
set( GDAL_DATA "${XDK_PATH}/share/gdal" )
set( GEOTIFF_CSV "${XDK_PATH}/share/epsg_csv" )
set( PROJ_LIB "${XDK_PATH}/share" )
set( CTEST_ENVIRONMENT
"PATH=$ENV{PATH}
")
endif()
include( "${CMAKE_CURRENT_LIST_DIR}/main_ci.cmake" )
......@@ -24,4 +24,6 @@ set(site_option
"OTB_USE_GLUT=OFF
OTB_USE_GLFW=OFF
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache")
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
OTB_DATA_USE_LARGEINPUT:BOOL=ON")
# Large input path are in an environment variable on macOS
......@@ -27,10 +27,22 @@ get_filename_component(OTB_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
set ( SUPERBUILD_SOURCE_DIR "${OTB_SOURCE_DIR}/SuperBuild" )
set ( CTEST_BUILD_CONFIGURATION "Release" )
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
if(WIN32)
set ( CTEST_CMAKE_GENERATOR "NMake Makefiles JOM" )
set ( CTEST_BUILD_FLAGS "/S" )
else()
set ( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
set ( CTEST_BUILD_FLAGS "-j16")
endif()
set ( PROJECT_SOURCE_DIR "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_SOURCE_DIRECTORY "${SUPERBUILD_SOURCE_DIR}" )
set ( CTEST_BINARY_DIRECTORY "${OTB_SOURCE_DIR}/build/" )
# Detect site
if(NOT DEFINED IMAGE_NAME)
if(DEFINED ENV{IMAGE_NAME})
set(IMAGE_NAME $ENV{IMAGE_NAME})
endif()
endif()
set ( CTEST_SITE "${IMAGE_NAME}" )
......@@ -48,9 +60,16 @@ set (CTEST_INSTALL_DIRECTORY "${OTB_SOURCE_DIR}/xdk/")
# HACK
# This is needed because when using return() function ctest is trying
# to run the CTEST_COMMAND. And we need it to not produce an error
set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
##set (CTEST_COMMAND "echo \"Exit\"") # HACK FIX ME
set (CMAKE_COMMAND "cmake")
if(WIN32)
file(TO_NATIVE_PATH "${CTEST_INSTALL_DIRECTORY}" XDK_INSTALL_DIR_NATIVE)
set(ENV{PATH} "${XDK_INSTALL_DIR_NATIVE}\\bin;$ENV{PATH}" )
set(ENV{PATH} "$ENV{PATH};${XDK_INSTALL_DIR_NATIVE}\\lib" )
# set(ENV{CMAKE_PREFIX_PATH} "${XDK_INSTALL_DIR}" )
endif()
########################################################################
########################################################################
# Build process
......@@ -64,14 +83,12 @@ find_program(CTEST_GIT_COMMAND NAMES git git.cmd)
set( GIT "${CTEST_GIT_COMMAND}" )
# Sources are already checked out : do nothing for update
set(CTEST_GIT_UPDATE_CUSTOM echo No update)
set(CTEST_GIT_UPDATE_CUSTOM "${CMAKE_COMMAND}" "-E" "echo" "No update")
ctest_start (Experimental TRACK CI_Prepare)
ctest_update( SOURCE "${OTB_SOURCE_DIR}" )
set(CTEST_BUILD_FLAGS "-j16")
set ( SB_CONFIGURE_OPTIONS "")
include( "${CMAKE_CURRENT_LIST_DIR}/sb_configure_options.cmake" )
......@@ -84,8 +101,7 @@ ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}"
if ( NOT _configure_rv EQUAL 0 )
ctest_submit()
message( SEND_ERROR "An error occurs during ctest_configure. Dependencies might be buggy.")
return()
message( FATAL_ERROR "An error occurs during ctest_configure. Dependencies might be buggy.")
endif()
########################################################################
......@@ -130,160 +146,167 @@ execute_process(
)
if ( IS_SB_BUILD )
message( "Superbuild is already build for ${IMAGE_NAME} with sources as ${SB_MD5}")
return()
else()
message( "No build available, this job will build and push OTB_DEPENDS")
endif()
####################################
# Back to build
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET "OTB_DEPENDS"
RETURN_VALUE _build_rv
NUMBER_ERRORS _build_nb_err
CAPTURE_CMAKE_ERROR _build_error
)
if ( DEBUG )
message( "Status for build:" )
message("_build_rv=${_build_rv}")
message("_build_nb_err=${_build_nb_err}")
message("_build_error=${_build_error}")
endif()
if ( ( NOT ${_build_nb_err} EQUAL 0 ) OR ( ${_build_error} EQUAL -1 ))
message( FATAL_ERROR "An error occurs during ctest_build.")
endif()
ctest_submit()
########################################################################
########################################################################
# Git process
########################################################################
########################################################################
# WE PUSH ONLY IF BUILD SUCCEED
# The image used will be passed to this script.
# TODO verify that images does not have forbidden char in there name
# TODO right now we rely on ctest_build to know whether there has been an error
# in build, whereas SuperBuild does not necessarily return an error if something
# goes wrong
# REPOSITORY_GIT_URL and REMOTE whould be the same. Right now there are
# different because one is https and one is ssh. Both should be ssh.
set( REPOSITORY_GIT_URL "git@gitlab.orfeo-toolbox.org:gbonnefille/superbuild-artifact.git")
# We clone master to have a basic configuration, mainly a correct .gitattribute
# git clone $REMOTE --branch master --depth 1 superbuild-artifact
execute_process(
COMMAND ${GIT} "clone" "${REPOSITORY_GIT_URL}"
"--branch" "master" "--depth" "1" "superbuild-artifact"
WORKING_DIRECTORY "${OTB_SOURCE_DIR}"
)
set ( SB_ARTIFACT_GIT "${OTB_SOURCE_DIR}/superbuild-artifact" )
# create a branche
execute_process(
COMMAND ${GIT} "checkout" "-b" "${BRANCH_NAME}"
WORKING_DIRECTORY ${SB_ARTIFACT_GIT}
RESULT_VARIABLE co_res
OUTPUT_VARIABLE co_out
ERROR_VARIABLE co_err
)
if ( DEBUG )
message( "Step 4: check-o")
message( "co_res = ${co_res}" )
message( "co_out = ${co_out}" )
message( "co_err = ${co_err}" )
endif()
set ( SB_TAR_NAME "SuperBuild_Install.tar" )
# Creating the tar
# May be for easier maintainability the tar name should be the same as the
# file inside.
execute_process(
COMMAND ${CMAKE_COMMAND} "-E" "tar" "cf" "${SB_ARTIFACT_GIT}/${SB_TAR_NAME}"
-- "${CTEST_INSTALL_DIRECTORY}"
WORKING_DIRECTORY ${OTB_SOURCE_DIR}
)
####################################
# Back to build
ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}"
TARGET "OTB_DEPENDS"
RETURN_VALUE _build_rv
NUMBER_ERRORS _build_nb_err
CAPTURE_CMAKE_ERROR _build_error