Commit 6d91a046 authored by Antoine Regimbeau's avatar Antoine Regimbeau

Merge branch 'develop' into 1834-test-data-lfs

parents da43153d a10f120f
......@@ -6,7 +6,6 @@ stages:
- build
.general:
only: [merge_requests]
retry:
max: 2
when:
......@@ -15,14 +14,66 @@ stages:
native-build:
extends: .general
only: [merge_requests]
stage: build
image: registry.orfeo-toolbox.org/gbonnefille/otb-build-env/otb-ubuntu-native:18.04
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc
shark-build:
.build-common:
extends: .general
stage: build
image: registry.orfeo-toolbox.org/gbonnefille/otb-build-env/otb-ubuntu-shark:18.04
before_script:
# ccache configuration
- mkdir -p ccache
- export CCACHE_BASEDIR=${PWD}
- export CCACHE_DIR=${PWD}/ccache
- echo "max_size = 3G" > ccache/ccache.conf
- echo "run_second_cpp = true" >> ccache/ccache.conf
- echo "compiler_check = content" >> ccache/ccache.conf
- echo "sloppiness = time_macros" >> ccache/ccache.conf
script:
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-gcc-shark-3.1.4
- ctest -VV -S CI/main_ci.cmake -DIMAGE_NAME:string=ubuntu-18.04-llvm-shark
cache:
key: "ubuntu-18.04-llvm-shark"
paths:
- ccache/
build-wip:
extends: .build-common
variables:
OTB_CI_PROFILE: 'wip'
except:
- merge_requests
- develop
- /^release-[0-9]+\.[0-9]+$/
cache:
policy: pull
build-mr:
extends: .build-common
variables:
OTB_CI_PROFILE: 'mr'
only:
- merge_requests
cache:
policy: pull
build-develop:
extends: .build-common
variables:
OTB_CI_PROFILE: 'develop'
only:
- develop
cache:
policy: pull-push
build-release:
extends: .build-common
variables:
OTB_CI_PROFILE: 'release'
only:
- /^release-[0-9]+\.[0-9]+$/
cache:
policy: pull
......@@ -39,7 +39,6 @@ Once all blocking issues are closed, and the previous steps are done:
* [ ] Application online documentation
* [ ] WordPress page "Home" and "Download" pages
* [ ] Upload OTB source archive to [Zenodo](https://zenodo.org/) to create a unique Digital Object Identifier (DOI)
* [ ] Update OTB-Data-Examples.tgz on orfeo-toolbox (packages)
* [ ] Send email to mailing list to announce the release
* [ ] Release announcement on the blog
* [ ] Announcement on social networks (twitter, google+)
......
......@@ -26,7 +26,17 @@ set (ENV{LANG} "C") # Only ascii output
set (CTEST_BUILD_CONFIGURATION "Release")
set (CTEST_CMAKE_GENERATOR "Ninja")
set (CTEST_BUILD_NAME "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}_to_$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
# Find the build name
set(ci_mr_source "$ENV{CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}")
set(ci_mr_target "$ENV{CI_MERGE_REQUEST_TARGET_BRANCH_NAME}")
set(ci_ref_name "$ENV{CI_COMMIT_REF_NAME}")
set (CTEST_BUILD_NAME "$ENV{CI_COMMIT_SHORT_SHA}")
if(ci_mr_source AND ci_mr_target)
set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME} (${ci_mr_source} to ${ci_mr_target})")
elseif(ci_ref_name)
set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME} (${ci_ref_name})")
endif()
set (CTEST_SITE "${IMAGE_NAME}")
# Directory variable
......@@ -41,6 +51,7 @@ set (CMAKE_COMMAND "cmake")
# Data directory setting
set (OTB_LARGEINPUT_ROOT "") # todo
message(STATUS "CI profile : $ENV{OTB_CI_PROFILE}")
#The following file set the CONFIGURE_OPTIONS variable
set (CONFIGURE_OPTIONS "")
......
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Configuration options for ubuntu-18.04-llvm-shark-3.1.4
set(site_option
"opencv_INCLUDE_DIR:PATH=/usr/include
CMAKE_C_COMPILER:STRING=clang
CMAKE_CXX_COMPILER:STRING=clang++
CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache")
#
# Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
#
# This file is part of Orfeo Toolbox
#
# https://www.orfeo-toolbox.org/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Configuration options for ubuntu-18.04-llvm
set(site_option
"opencv_INCLUDE_DIR:PATH=/usr/include
OTB_USE_SHARK:BOOL=OFF
CMAKE_C_COMPILER:STRING=clang
CMAKE_CXX_COMPILER:STRING=clang++
CMAKE_EXE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_MODULE_LINKER_FLAGS:STRING=-fuse-ld=lld
CMAKE_SHARED_LINKER_FLAGS:STRING=-fuse-ld=lld")
......@@ -63,7 +63,6 @@ set(CTEST_CUSTOM_COVERAGE_EXCLUDE
# Exclude SWIG wrappers files
".*/Modules/Wrappers/SWIG/src/python/otbApplicationPYTHON_wrap.*"
".*/Modules/Wrappers/SWIG/src/python3/otbApplicationPYTHON_wrap.*"
".*/Modules/Wrappers/SWIG/src/java/otbApplicationJAVA_wrap.*"
# Exclude Qt moc file
......
......@@ -26,7 +26,6 @@
# python is needed to verify the presence of the module name in the doxygen header
# Don't require it to not force the developers to install python to be able to build
# OTB. The tests will simply not be run if python is not available.
find_package(PythonInterp)
macro( otb_module_doxygen _name )
......
......@@ -26,7 +26,6 @@
# This does not force the developer to install python to be able to build OTB.
# The tests will simply not be run if python is unavailable.
find_package(PythonInterp)
# The maximum number of headers in a test. This helps limit memory issues,
# and the cppcheck tests. However, if this is not unity, there is a slight
......
......@@ -100,6 +100,19 @@ reset_qt_i18n_sources()
repository_status(${PROJECT_SOURCE_DIR} OTB_GIT_STATUS_MESSAGE)
# Find python stuff
# Version 3 is prefered before 2
set ( Python_ADDITIONAL_VERSIONS "3;2" )
set ( PythonInterp_FIND_REQUIRED ${OTB_WRAP_PYTHON} )
set ( PythonLibs_FIND_REQUIRED ${OTB_WRAP_PYTHON} )
find_package( PythonInterp )
if ( OTB_WRAP_PYTHON AND ( ${PYTHON_VERSION_MAJOR} EQUAL 2 ) )
message (WARNING "Python3 not found. There is no longer support of \
wrapping in python2 in OTB, but it can still be used.")
endif()
set (Python_ADDITIONAL_VERSIONS "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}")
find_package ( PythonLibs )
#-----------------------------------------------------------------------------
# Configure the default OTB_DATA_ROOT for the location of OTB Data.
find_path(OTB_DATA_ROOT
......@@ -152,9 +165,6 @@ endif()
if(NOT OTB_INSTALL_PYTHON_DIR)
set(OTB_INSTALL_PYTHON_DIR "${OTB_INSTALL_LIBRARY_DIR}/otb/python")
endif()
if(NOT OTB_INSTALL_PYTHON3_DIR)
set(OTB_INSTALL_PYTHON3_DIR "${OTB_INSTALL_LIBRARY_DIR}/otb/python3")
endif()
if(NOT OTB_INSTALL_JAVA_DIR)
set(OTB_INSTALL_JAVA_DIR "${OTB_INSTALL_LIBRARY_DIR}/otb/java")
endif()
......@@ -339,6 +349,7 @@ set(OTB_CONFIG_CODE "
set(OTB_MODULES_DIR \"${OTB_MODULES_DIR}\")")
set(OTB_CONFIG_CMAKE_DIR "${OTB_SOURCE_DIR}/CMake")
set(OTB_USE_FILE "${OTB_CONFIG_CMAKE_DIR}/UseOTB.cmake")
if(OTB_WRAP_PYTHON)
set(OTB_CONFIG_CODE "${OTB_CONFIG_CODE}
set(OTB_PYTHONPATH \"${OTB_BINARY_DIR}/${OTB_INSTALL_PYTHON_DIR}\")")
......@@ -366,10 +377,12 @@ set(OTB_CONFIG_CODE "${OTB_CONFIG_CODE}
set(OTB_MODULES_DIR \"\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_PACKAGE_DIR}/Modules\")")
set(OTB_USE_FILE "\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_PACKAGE_DIR}/UseOTB.cmake")
set(OTB_CONFIG_CMAKE_DIR "\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_PACKAGE_DIR}")
if(OTB_WRAP_PYTHON)
set(OTB_CONFIG_CODE "${OTB_CONFIG_CODE}
set(OTB_PYTHONPATH \"\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_PYTHON_DIR}\")")
endif()
set(OTB_CONFIG_APPLICATION_PATH "\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_APP_DIR}")
set(OTB_CONFIG_TARGETS_CONDITION "")
set(OTB_CONFIG_TARGETS_FILE "\${OTB_INSTALL_PREFIX}/${OTB_INSTALL_PACKAGE_DIR}/OTBTargets.cmake")
......
......@@ -66,9 +66,9 @@ foreach(cmd
endforeach()
# We need to set PYTHONPATH for the script otbGenerateWrappersRstDoc.py, depending on how the Python3 module was built
if (OTB_WRAP_PYTHON3)
set(PYTHONPATH_COOKBOOK "${CMAKE_BINARY_DIR}/lib/otb/python3")
elseif (OTB_WRAP_PYTHON)
if (OTB_WRAP_PYTHON)
set(PYTHONPATH_COOKBOOK "${CMAKE_BINARY_DIR}/lib/otb/python")
else()
# Cookbook only supports Python3
# But OTB_WRAP_PYTHON can wrap both python2 and python3
if (${PYTHONLIBS_VERSION_STRING} STRGREATER "3.0.0")
......
import argparse
import re
import os
import os.path
from os.path import join
import subprocess
def sed(content, regex, repl):
return re.sub(regex, repl, content, flags = re.MULTILINE | re.DOTALL)
if __name__ == "__main__":
parser = argparse.ArgumentParser(usage="migrate sg tex file")
parser.add_argument("filename", help="")
parser.add_argument("output_dir", help="")
args = parser.parse_args()
input = args.filename
output = join(args.output_dir, os.path.basename(args.filename).replace(".tex", ".rst"))
content = open(input).read()
content = sed(content,
r"\\doxygen\{otb\}\{(.*?)\}",
r":doxygen:`\1`")
content = sed(content,
r"\\doxygen\{itk\}\{(.*?)\}",
r":doxygen-itk:`\1`")
content = sed(content,
r"\\code\{(.*?)\}",
r"\\texttt{\1}")
content = sed(content,
r"cmakecode",
r"verbatim")
content = sed(content,
r"cppcode",
r"verbatim")
content = sed(content,
r"\\input\{(.*?)\}",
r"See example :ref:`\1`")
content = sed(content,
r"\\input (\w+)\n",
r"See example \1\n")
content = sed(content,
r"\\begin\{figure\}",
r"\\begin{verbatim}\\begin{figure}")
content = sed(content,
r"\\end\{figure\}",
r"\\end{figure}\\end{verbatim}")
open(output, "w").write(content)
subprocess.check_call("pandoc -f latex -t rst -o {} {}".format(output, output), shell=True)
subprocess.check_call(['sed', '-i', "s ‘ ` g", output])
print(output)
......@@ -95,9 +95,10 @@ def render_example(filename, otb_root):
rst_description = ""
# Render the template
name = os.path.basename(filename)
template_example = open("templates/example.rst").read()
output_rst = template_example.format(
label="example-" + root,
label=name,
heading=rst_section(name, "="),
description=rst_description,
usage=example_usage,
......@@ -108,7 +109,7 @@ def render_example(filename, otb_root):
return output_rst
if __name__ == "__main__":
def main():
parser = argparse.ArgumentParser(usage="Export examples to rst")
parser.add_argument("rst_dir", help="Directory where rst files are generated")
parser.add_argument("otb_root", help="OTB repository root")
......@@ -130,3 +131,6 @@ if __name__ == "__main__":
os.makedirs(join(args.rst_dir, "C++", "Examples", tag), exist_ok=True)
with open(join(args.rst_dir, "C++", "Examples", tag, root + ".rst"), "w") as output_file:
output_file.write(render_example(filename, args.otb_root))
if __name__ == "__main__":
main()
......@@ -360,7 +360,7 @@ def multireplace(string, replacements):
def make_links(text, allapps):
"Replace name of applications by internal rst links"
rep = {appname: ":ref:`{}`".format("app-" + appname) for appname in allapps}
rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
return multireplace(text, rep)
def render_application(appname, allapps):
......@@ -374,7 +374,7 @@ def render_application(appname, allapps):
application_documentation_warnings(app)
output = template_application.format(
label="app-" + appname,
label=appname,
heading=rst_section(app.GetName(), '='),
description=app.GetDescription(),
longdescription=make_links(app.GetDocLongDescription(), allapps),
......@@ -404,7 +404,7 @@ def GenerateRstForApplications(rst_dir):
appNames = [app for app in allApps if app not in blackList]
appIndexFile = open(rst_dir + '/Applications.rst', 'w')
appIndexFile.write(RstPageHeading("Applications", "2", ref="apprefdoc"))
appIndexFile.write(RstPageHeading("All Applications", "2", ref="apprefdoc"))
print("Generating rst for {} applications".format(len(appNames)))
......
.wy-nav-content {
max-width: 800px;
/* Style of the toctree captions
* inspired from sphinx-rtd-theme docs
*/
.wy-menu-vertical header, .wy-menu-vertical p.caption {
color: #2980b9; /* sphinx blue */
line-height: 32px;
font-weight: bold;
text-transform: uppercase;
white-space: nowrap;
margin-top: .8em;
}
/* Reduce the effect of the p bottom margin before lists
* Very useful for choice parameters in app doc for example
*/
p + ul {
.wy-nav-content .document p + ul {
margin-top: -18px;
}
/* Custom colors for the logo background */
.wy-side-nav-search, .wy-nav-top {
background: #1e1f22; /* otb black */
}
.wy-side-nav-search input[type="text"] {
border-color: #1e1f22; /* otb black */
}
......@@ -4,5 +4,14 @@ C++ API
=======
.. toctree::
:maxdepth: 2
C++/SystemOverview.rst
C++/Tutorial.rst
C++/Iterators.rst
C++/Filters.rst
C++/StreamingAndThreading.rst
C++/PersistentFilters.rst
C++/WriteAnApplication.rst
C++/AddingNewModules.rst
C++/Examples.rst
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Persistent filters
==================
Introduction
------------
As presented in chapter :ref:`StreamingAndThreading`, OTB has two main mechanisms
to handle large data: streaming allows to process image piece-wise, and
multi-threading allows to process concurrently several pieces of one streaming
block. Using these concepts, one can easily write pixel-wise or
neighborhood-based filters and insert them into a pipeline which will be
scalable with respect to the input image size.
Yet, sometimes we need to compute global features on the whole image.
One example is to determine image mean and variance of the input image
in order to produce a centered and reduced image. The operation of
centering and reducing each pixel is fully compliant with streaming and
threading, but one has to first estimate the mean and variance of the
image. This first step requires to walk the whole image once, and
traditional streaming and multi-threading based filter architecture is
of no help here.
This is because there is a fundamental difference between these two
operations: one supports streaming, and the other needs to perform
streaming. In fact we would like to stream the whole image piece by
piece through some filter that will collect and keep mean and variance
cumulants, and then synthetize theses cumulants to compute the final
mean and variance once the full image as been streamed. Each stream
would also benefit from parallel processing. This is exactly what
persistent filters are for.
Architecture
------------
There are two main objects in the persistent filters framework. The
first is the :doxygen:`PersistentImageFilter`, the second is the
:doxygen:`PersistentFilterStreamingDecorator`.
The persistent filter class
~~~~~~~~~~~~~~~~~~~~~~~~~~~
The :doxygen:`PersistentImageFilter` class is a regular
:doxygen-itk:`ImageToImageFilter`, with two additional pure virtual
methods: the ``Synthetize()`` and the ``Reset()`` methods.
Imagine that the ``GenerateData()`` or ``ThreadedGenerateData()``
progressively computes some global feature of the whole image, using
some member of the class to store intermediate results. The
``Synthetize()`` is an additional method which is designed to be called
one the whole image has been processed, in order to compute the final
results from the intermediate results. The ``Reset()`` method is
designed to allow the reset of the intermediate results members so as to
start a fresh processing.
Any sub-class of the :doxygen:`PersistentImageFilter` can be used as a
regular :doxygen-itk:`ImageToImageFilter` (provided that both
``Synthetize()`` and ``Reset()`` have been implemented, but the real
interest of these filters is to be used with the streaming decorator
class presented in the next section.
The streaming decorator class
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The :doxygen:`PersistentFilterStreamingDecorator` is a class designed to
be templated with subclasses of the :doxygen:`PersistentImageFilter`. It
provides the mechanism to stream the whole image through the templated
filter, using a third class called
:doxygen:`StreamingImageVirtualWriter`. When the ``Update()`` method is
called on a :doxygen:`PersistentFilterStreamingDecorator`, a pipeline
plugging the templated subclass of the :doxygen:`PersistentImageFilter`
to an instance of :doxygen:`StreamingImageVirtualWriter` is created. The
latter is then updated, and acts like a regular
:doxygen:`ImageFileWriter` but it does not actually write anything to
the disk : streaming pieces are requested and immediately discarded. The
:doxygen:`PersistentFilterStreamingDecorator` also calls the ``Reset()``
method at the beginning and the ``Synthetize()`` method at the end of
the streaming process. Therefore, it packages the whole mechanism for
the use of a :doxygen:`PersistentImageFilter`:
#. Call the ``Reset()`` method on the filter so as to reset any
temporary results members,
#. Stream the image piece-wise through the filter,
#. Call the ``Synthetize()`` method on the filter so as to compute the
final results.
There are some methods that allows to tune the behavior of the
:doxygen:`StreamingImageVirtualWriter`, allowing to change the image
splitting methods (tiles or strips) or the size of the streams with
respect to some target available amount of memory. Please see the class
documentation for details. The instance of the
:doxygen:`StreamingImageVirtualWriter` can be retrieved from the
:doxygen:`PersistentFilterStreamingDecorator` through the
``GetStreamer()`` method.
Though the internal filter of the
:doxygen:`PersistentFilterStreamingDecorator` can be accessed through
the ``GetFilter()`` method, the class is often derived to package the
streaming-decorated filter and wrap the parameters setters and getters.
An end-to-end example
---------------------
This is an end-to-end example to compute the mean over a full image,
using a streaming and threading-enabled filter. Please note that only
specific details are explained here. For more general information on how
to write a filter, please refer to section :ref:`Filters`.
First step: writing a persistent filter
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The first step is to write a persistent mean image filter. We need to
include the appropriate header :
.. code-block:: cpp
#include "otbPersistentImageFilter.h"
Then, we declare the class prototype as follows:
.. code-block:: cpp
template<class TInputImage>
class ITK_EXPORT PersistentMeanImageFilter :
public PersistentImageFilter<TInputImage, TInputImage>
Since the output image will only be used for streaming purpose, we do
not need to declare different input and output template types.
In the *private* section of the class, we will declare a member which
will be used to store temporary results, and a member which will be used
to store the final result.
.. code-block:: cpp
private:
// Temporary results container
std::vector<PixelType> m_TemporarySums;
// Final result member
double m_Mean;
Next, we will write the ``Reset()`` method implementation in the
*protected* section of the class. Proper allocation of the temporary
results container with respect to the number of threads is handled here.
.. code-block:: cpp
protected:
virtual void Reset()
{
// Retrieve the number of threads
unsigned int numberOfThreads = this->GetNumberOfThreads();
// Reset the temporary results container
m_TemporarySums = std::vector<PixelType>(numberOfThreads, itk::NumericTraits<PixelType>::Zero);
// Reset the final result
m_Mean = 0.;
}
Now, we need to write the ``ThreadedGenerateData()`` methods (also in
the *protected* section), were temporary results will be computed for
each piece of stream.
.. code-block:: cpp
virtual void ThreadedGenerateData(const RegionType&
outputRegionForThread,
itk::ThreadIdType threadId)
{
// Enable progress reporting
itk::ProgressReporter(this,threadId,outputRegionForThread.GetNumberOfPixels());
// Retrieve the input pointer
InputImagePointer inputPtr = const_cast<TInputImage *>(this->GetInput());
// Declare an iterator on the region
itk::ImageRegionConstIteratorWithIndex<TInputImage> it(inputPtr,
outputRegionForThread);
// Walk the region of the image with the iterator
for (it.GoToBegin(); !it.IsAtEnd(); ++it, progress.CompletedPixel())
{
// Retrieve pixel value
const PixelType& value = it.Get();
// Update temporary results for the current thread
m_TemporarySums[threadId]+= value;
}
}
Last, we need to define the ``Synthetize()`` method (still in the
*protected* section), which will yield the final results:
.. code-block:: cpp
virtual void Synthetize()
{
// For each thread
for(unsigned int threadId = 0; threadId <this->GetNumberOfThreads();++threadId)
{
// Update final result
m_Mean+=m_TemporarySums[threadId];
}
// Complete calculus by dividing by the total number of pixels
unsigned int nbPixels = this->GetInput()->GetLargestPossibleRegion().GetNumberOfPixels();
if(nbPixels != 0)
{
m_Mean /= nbPixels;
}
}
Second step: Decorating the filter and using it
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Now, to use the filter, one only has to decorate it with the
:doxygen:`PersistentFilterStreamingDecorator`. First step is to include
the appropriate header:
.. code-block:: cpp
#include "otbPersistentMeanImageFilter.h"