Commit cf32d051 authored by Antoine Regimbeau's avatar Antoine Regimbeau

Merge branch 'develop' into 1506-refactor-MapProjectionAdapter

parents 2c333f98 943cfaa5
Pipeline #453 passed with stage
in 32 minutes and 44 seconds
......@@ -6,7 +6,6 @@ stages:
- build
.general:
only: [merge_requests]
retry:
max: 2
when:
......@@ -17,12 +16,13 @@ stages:
native-build:
extends: .general
only: [merge_requests]
stage: build
image: registry.orfeo-toolbox.org/gbonnefille/otb-build-env/otb-ubuntu-native:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
shark-build:
build:ubuntu-llvm:
extends: .general
stage: build
image: registry.orfeo-toolbox.org/gbonnefille/otb-build-env/otb-ubuntu-shark:18.04
......
......@@ -39,7 +39,6 @@ Once all blocking issues are closed, and the previous steps are done:
* [ ] Application online documentation
* [ ] WordPress page "Home" and "Download" pages
* [ ] Upload OTB source archive to [Zenodo](https://zenodo.org/) to create a unique Digital Object Identifier (DOI)
* [ ] Update OTB-Data-Examples.tgz on orfeo-toolbox (packages)
* [ ] Send email to mailing list to announce the release
* [ ] Release announcement on the blog
* [ ] Announcement on social networks (twitter, google+)
......
......@@ -26,7 +26,24 @@ set (ENV{LANG} "C") # Only ascii output
set (CTEST_BUILD_CONFIGURATION "Release")
set (CTEST_CMAKE_GENERATOR "Ninja")
set (CTEST_BUILD_NAME "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}_to_$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
# Find the build name and CI profile
set(ci_profile wip)
set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
set (CTEST_BUILD_NAME "$ENV{CI_COMMIT_SHORT_SHA}")
if(ci_mr_source AND ci_mr_target)
set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME} (${ci_mr_source} to ${ci_mr_target})")
set(ci_profile mr)
elseif(ci_ref_name)
set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME} (${ci_ref_name})")
if("${ci_ref_name}" STREQUAL "develop")
set(ci_profile develop)
elseif("${ci_ref_name}" MATCHES "^release-[0-9]+\\.[0-9]+\$")
set(ci_profile release)
endif()
endif()
set (CTEST_SITE "${IMAGE_NAME}")
# Directory variable
......@@ -42,6 +59,7 @@ set (CMAKE_COMMAND "cmake")
set (OTB_DATA_ROOT "${OTB_SOURCE_DIR}/otb-data/") # todo
set (OTB_LARGEINPUT_ROOT "") # todo
message(STATUS "CI profile : ${ci_profile}")
#The following file set the CONFIGURE_OPTIONS variable
set (CONFIGURE_OPTIONS "")
......
......@@ -26,4 +26,6 @@ CMAKE_C_COMPILER:STRING=clang
CMAKE_CXX_COMPILER:STRING=clang++
CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld")
CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache")
import argparse
import re
import os
import os.path
from os.path import join
import subprocess
def sed(content, regex, repl):
return re.sub(regex, repl, content, flags = re.MULTILINE | re.DOTALL)
if __name__ == "__main__":
parser = argparse.ArgumentParser(usage="migrate sg tex file")
parser.add_argument("filename", help="")
parser.add_argument("output_dir", help="")
args = parser.parse_args()
input = args.filename
output = join(args.output_dir, os.path.basename(args.filename).replace(".tex", ".rst"))
content = open(input).read()
content = sed(content,
r"\\doxygen\{otb\}\{(.*?)\}",
r":doxygen:`\1`")
content = sed(content,
r"\\doxygen\{itk\}\{(.*?)\}",
r":doxygen-itk:`\1`")
content = sed(content,
r"\\code\{(.*?)\}",
r"\\texttt{\1}")
content = sed(content,
r"cmakecode",
r"verbatim")
content = sed(content,
r"cppcode",
r"verbatim")
content = sed(content,
r"\\input\{(.*?)\}",
r"See example :ref:`\1`")
content = sed(content,
r"\\input (\w+)\n",
r"See example \1\n")
content = sed(content,
r"\\begin\{figure\}",
r"\\begin{verbatim}\\begin{figure}")
content = sed(content,
r"\\end\{figure\}",
r"\\end{figure}\\end{verbatim}")
open(output, "w").write(content)
subprocess.check_call("pandoc -f latex -t rst -o {} {}".format(output, output), shell=True)
subprocess.check_call(['sed', '-i', "s ‘ ` g", output])
print(output)
......@@ -95,9 +95,10 @@ def render_example(filename, otb_root):
rst_description = ""
# Render the template
name = os.path.basename(filename)
template_example = open("templates/example.rst").read()
output_rst = template_example.format(
label="example-" + root,
label=name,
heading=rst_section(name, "="),
description=rst_description,
usage=example_usage,
......@@ -108,7 +109,7 @@ def render_example(filename, otb_root):
return output_rst
if __name__ == "__main__":
def main():
parser = argparse.ArgumentParser(usage="Export examples to rst")
parser.add_argument("rst_dir", help="Directory where rst files are generated")
parser.add_argument("otb_root", help="OTB repository root")
......@@ -130,3 +131,6 @@ if __name__ == "__main__":
os.makedirs(join(args.rst_dir, "C++", "Examples", tag), exist_ok=True)
with open(join(args.rst_dir, "C++", "Examples", tag, root + ".rst"), "w") as output_file:
output_file.write(render_example(filename, args.otb_root))
if __name__ == "__main__":
main()
......@@ -360,7 +360,7 @@ def multireplace(string, replacements):
def make_links(text, allapps):
"Replace name of applications by internal rst links"
rep = {appname: ":ref:`{}`".format("app-" + appname) for appname in allapps}
rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
return multireplace(text, rep)
def render_application(appname, allapps):
......@@ -374,7 +374,7 @@ def render_application(appname, allapps):
application_documentation_warnings(app)
output = template_application.format(
label="app-" + appname,
label=appname,
heading=rst_section(app.GetName(), '='),
description=app.GetDescription(),
longdescription=make_links(app.GetDocLongDescription(), allapps),
......@@ -404,7 +404,7 @@ def GenerateRstForApplications(rst_dir):
appNames = [app for app in allApps if app not in blackList]
appIndexFile = open(rst_dir + '/Applications.rst', 'w')
appIndexFile.write(RstPageHeading("Applications", "2", ref="apprefdoc"))
appIndexFile.write(RstPageHeading("All Applications", "2", ref="apprefdoc"))
print("Generating rst for {} applications".format(len(appNames)))
......
.wy-nav-content {
max-width: 800px;
/* Style of the toctree captions
* inspired from sphinx-rtd-theme docs
*/
.wy-menu-vertical header, .wy-menu-vertical p.caption {
color: #2980b9; /* sphinx blue */
line-height: 32px;
font-weight: bold;
text-transform: uppercase;
white-space: nowrap;
margin-top: .8em;
}
/* Reduce the effect of the p bottom margin before lists
* Very useful for choice parameters in app doc for example
*/
p + ul {
.wy-nav-content .document p + ul {
margin-top: -18px;
}
/* Custom colors for the logo background */
.wy-side-nav-search, .wy-nav-top {
background: #1e1f22; /* otb black */
}
.wy-side-nav-search input[type="text"] {
border-color: #1e1f22; /* otb black */
}
......@@ -4,5 +4,14 @@ C++ API
=======
.. toctree::
:maxdepth: 2
C++/SystemOverview.rst
C++/Tutorial.rst
C++/Iterators.rst
C++/Filters.rst
C++/StreamingAndThreading.rst
C++/PersistentFilters.rst
C++/WriteAnApplication.rst
C++/AddingNewModules.rst
C++/Examples.rst
Adding New Modules
==================
This chapter is concerned with the creation of new modules. The
following sections give precise instructions about:
- the organization of directories
- the included files
- what they must contain
- ...
How to Write a Module
---------------------
There is a template of OTB remote module which help you start developing
a remote module: `External Module
Template <https://gitlab.orfeo-toolbox.org/remote_modules/remote-module-template>`__.
Each module is made of different components, described in the
following sections.
The otb-module.cmake file
-------------------------
This file is mandatory. It follows the CMake syntax, and has two
purposes:
- Declare dependencies to other modules,
- Provide a short description of the module purpose.
These purposes are fulfilled by a single CMake Macro call:
.. code-block:: cmake
otb_module(TheModuleName DEPENDS OTBModule1 OTBModule2 ... OTBModuleN DESCRIPTION "A description string")
**Note**: You can use the keyword ``TESTDEPENDS`` to declare module
dependencies that only applies to the tests.
The CMakeLists.txt file
-----------------------
The ``CMakeLists.txt`` file is mandatory. It contains only a few things.
First, it declares a new CMake project with the name of the module:
.. code-block:: cmake
project(TheModuleName)
Second, if the module contain a library (see src folder section below),
it initializes the TheModuleNameLIBRARIES CMake variable (if your module
only contains headers or template code, skip this line):
.. code-block:: cmake
set(TheModuleName_LIBRARIES OTBTheModuleName)
You can build your remote modules inside the OTB source tree by copying
your source inside the directory ``Module/Remote`` or against an existing
OTB build tree (note that it does not work with an install version of
OTB).
The configuration below will handle both cases and take care of all the
CMake plumbing of the module:
.. code-block:: cmake
if(NOT OTB_SOURCE_DIR)
find_package(OTB REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${OTB_CMAKE_DIR})
include(OTBModuleExternal)
else()
otb_module_impl()
endif()
The overall file should look like this:
.. code-block:: cmake
cmake_minimum_required(VERSION 2.8.9)
project(TheModuleName)
set(ExternalTemplate_LIBRARIES OTBTheModuleName)
if(NOT OTB_SOURCE_DIR)
find_package(OTB REQUIRED)
list(APPEND CMAKE_MODULE_PATH ${OTB_CMAKE_DIR})
include(OTBModuleExternal)
else()
otb_module_impl()
endif()
The include folder
------------------
The include folder will contain all your headers (``*.h`` files) and
template method files (``*.hxx`` or ``*.hxx``). It does not require any
additional file (in particular, no CMakeLists.txt file is required).
The src folder
--------------
The src folder contains the internal implementation of your module:
- It typically contains cxx source files that will be compiled into a
library.
- It can contain header files for classes used only within the
implementation files of your module. Any header file present in the
src folder will not be installed, and will not be available to other
modules depending on your module.
If your modules is made of template only code, you do not need a src
folder at all.
If present, the src folder requires a CMakeLists.txt file.
The first part of the CMakeLists.txt file is classical, as it builds the
library and links it:
.. code-block:: cmake
set(OTBTheModuleName_SRC
sourceFile1.cxx
sourceFile2.cxx
sourceFile3.cxx
...
sourceFileN.cxx)
add_library(OTBTheModuleName ${OTBTheModuleName_SRC})
target_link_libraries(OTBTheModuleName ${OTBModule1_LIBRARIES} ${OTBModule2_LIBRARIES} ... ${OTBModuleN_LIBRARIES})
**Notes**:
- Library name should match the one declared in the root CMakeLists.txt
when setting CMake variable TheModuleNameLIBRARIES,
- Linked libraries should match the dependencies of your module
declared in the root otb-module.cmake file.
The last line of CMake code takes care of installation instructions:
.. code-block:: cmake
otb_module_target(TBTheModuleName)
The overall CMakeLists.txt file should look like:
.. code-block:: cmake
set(OTBTheModuleName_SRC
sourceFile1.cxx
sourceFile2.cxx
sourceFile3.cxx
...
sourceFileN.cxx)
add_library(OTBTheModuleName ${OTBTheModuleName_SRC})
target_link_libraries(OTBTheModuleName ${OTBModule1_LIBRARIES} ${OTBModule2_LIBRARIES} ... ${OTBModuleN_LIBRARIES})
otb_module_target(TBTheModuleName)
The app folder
--------------
The app folder contains the code of applications shipped with your
module. If your module has no application, you do not need the app
folder.
**Notes**: If your module contains application (and an app folder), do
not forget to add the ApplicationEngine in the dependencies listed in
the otb-module.cmake file.
In addition to the applications source code, the app folder should
contain a CMakeLists.txt file as follows.
For each application, a single call otbcreateapplication is required:
.. code-block:: cmake
otb_create_application(
NAME TheModuleApplication1
SOURCES TheModuleApplication1.cxx
LINK_LIBRARIES ${OTBModule1_LIBRARIES} ${OTBModule2_LIBRARIES} ... ${OTBModuleN_LIBRARIES})
The test folder
---------------
This folder contains tests of the module. If your module has no test in
it (which is not recommended, you do not need it).
The test folder should contain the source files of tests, as well as a
CMakeLists.txt file. This file will contain the following.
First, indicate that this folder contains tests.
.. code-block:: cmake
otb_module_test()
Then, build the test driver:
.. code-block:: cmake
set(OTBTheModuleNameTests
testFile1.cxx
testFile2.cxx
...
testFileN.cxx)
add_executable(otbTheModuleNameTestDriver ${OTBTheModuleNameTests})
target_link_libraries(otbTheModuleNameTestDriver ${OTBTheModuleName-Test_LIBRARIES})
otb_module_target_label(otbTheModuleNameTestDriver)
Finally, you can add your tests:
.. code-block:: cmake
otb_add_test(NAME nameOfTheTest COMMAND otbTheModuleNameTestDriver
--compare-image ${EPSILON_8} ... # baseline comparison if needed
nameOfTheTestFunction
testParameters)
If your module contains one or more application in the app folder, you
should also write tests for them, in the test folder. Running an
application test is easily done with the helper macro
otbtestapplication:
.. code-block:: cmake
otb_test_application(NAME nameofApplication1Test1
APP TheModuleApplication1
OPTIONS -in1 ${INPUTDATA}/input1.tif
-in2 ${INPUTDATA}/input2.tif
-out ${TEMP}/nameofApplication1Test1_result.tif
VALID --compare-image ${EPSILON_8}
${BASELINE}/nameofApplication1Test1_result.tif
${TEMP}/nameofApplication1Test1_result.tif)
Overall CMakeLists.txt should look like:
.. code-block:: cmake
otb_module_test()
set(OTBTheModuleNameTests
testFile1.cxx
testFile2.cxx
...
testFileN.cxx)
add_executable(otbTheModuleNameTestDriver ${OTBTheModuleNameTests})
target_link_libraries(otbTheModuleNameTestDriver ${OTBTheModuleName-Test_LIBRARIES})
otb_module_target_label(otbTheModuleNameTestDriver)
otb_add_test(NAME nameOfTheTest COMMAND otbTheModuleNameTestDriver
--compare-image ${EPSILON_8} ... # baseline comparison if needed
nameOfTheTestFunction
testParameters)
Including a remote module in OTB
--------------------------------
* Local build of a remote module
Your remote module can be build inside the OTB source tree or outside as
a external CMake project with an existing OTB. Please note in that case
that you’ll have to set OTBDIR CMake option.
If OTBDIR is an OTB build tree, there are two ways of compiling:
- Build as a module, in which case build files will be written to the
OTB build tree as other modules. Main benefit is that this will
enrich the current OTB build with your new module, but you need to
have write access to the build directory.
- Build as a standalone CMake project, in which case build files will
remain in the module build folder. This build is fully independent
from the build (or install) directory, but the module will not be
recognized as an OTB module (still you will be able to use its
binaries and libraries).
This behaviour is controlled by the ``OTB_BUILD_MODULE_AS_STANDALONE``, which is
OFF by default (hence first behaviour).
Note that when dealing with an installed OTB, only the second behaviour
(build as standalone) is available.
Optionally, you can build your new remote module inside the OTB source
tree by simply copy the folder containing the module component to
Modules/Remote, then run CMake configuration. you should see a new CMake
option named MODULETheModuleName. Simply turn this option to ON, and
finish CMake configuration. Your module will be built with the rest of
the OTB project.
* Sharing your remote module
To make your remote module available to others when building OTB, you
should provide a CMake file named TheModuleName.remote.cmake file for
inclusion in the Modules/Remote folder in OTB source tree.
This file should contain the following:
.. code-block:: cmake
# Contact: Author name <author email address>
otb_fetch_module(TheModuleName
"A description of the module, to appear during CMake configuration step"
GIT_REPOSITORY http_link_to_a_git_repository_hosting the module
GIT TAG the git revision to checkout
)
This file should be provided along with your remote module inclusion
proposal email to the otb community list. Please refer to the
contributors guidelines for more information.
This diff is collapsed.
This diff is collapsed.
Persistent filters
==================
Introduction
------------
As presented in chapter :ref:`StreamingAndThreading`, OTB has two main mechanisms
to handle large data: streaming allows to process image piece-wise, and
multi-threading allows to process concurrently several pieces of one streaming
block. Using these concepts, one can easily write pixel-wise or
neighborhood-based filters and insert them into a pipeline which will be
scalable with respect to the input image size.
Yet, sometimes we need to compute global features on the whole image.
One example is to determine image mean and variance of the input image
in order to produce a centered and reduced image. The operation of
centering and reducing each pixel is fully compliant with streaming and
threading, but one has to first estimate the mean and variance of the
image. This first step requires to walk the whole image once, and
traditional streaming and multi-threading based filter architecture is
of no help here.
This is because there is a fundamental difference between these two
operations: one supports streaming, and the other needs to perform