Merge branch 'cookbook-devguide' into 'develop'

Migrate developer's guide to the cookbook

See merge request !398

Documentation/Cookbook/Scripts/migrate_sg_tex.py

+import argparse
+import re
+import os
+import os.path
+from os.path import join
+import subprocess
+
+def sed(content, regex, repl):
+    return re.sub(regex, repl, content, flags = re.MULTILINE | re.DOTALL)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(usage="migrate sg tex file")
+    parser.add_argument("filename", help="")
+    parser.add_argument("output_dir", help="")
+    args = parser.parse_args()
+
+    input = args.filename
+    output = join(args.output_dir, os.path.basename(args.filename).replace(".tex", ".rst"))
+
+    content = open(input).read()
+
+    content = sed(content,
+            r"\\doxygen\{otb\}\{(.*?)\}",
+            r":doxygen:`\1`")
+
+    content = sed(content,
+            r"\\doxygen\{itk\}\{(.*?)\}",
+            r":doxygen-itk:`\1`")
+
+    content = sed(content,
+            r"\\code\{(.*?)\}",
+            r"\\texttt{\1}")
+
+    content = sed(content,
+            r"cmakecode",
+            r"verbatim")
+
+    content = sed(content,
+            r"cppcode",
+            r"verbatim")
+
+    content = sed(content,
+            r"\\input\{(.*?)\}",
+            r"See example :ref:`\1`")
+
+    content = sed(content,
+            r"\\input (\w+)\n",
+            r"See example \1\n")
+
+    content = sed(content,
+            r"\\begin\{figure\}",
+            r"\\begin{verbatim}\\begin{figure}")
+
+    content = sed(content,
+            r"\\end\{figure\}",
+            r"\\end{figure}\\end{verbatim}")
+
+    open(output, "w").write(content)
+
+    subprocess.check_call("pandoc -f latex -t rst -o {} {}".format(output, output), shell=True)
+    subprocess.check_call(['sed', '-i', "s ‘ ` g", output])
+    print(output)

Documentation/Cookbook/Scripts/otbGenerateExamplesRstDoc.py

@@ -95,9 +95,10 @@ def render_example(filename, otb_root):
         rst_description = ""
 
     # Render the template
+    name = os.path.basename(filename)
     template_example = open("templates/example.rst").read()
     output_rst = template_example.format(
-        label="example-" + root,
+        label=name,
         heading=rst_section(name, "="),
         description=rst_description,
         usage=example_usage,
@@ -108,7 +109,7 @@ def render_example(filename, otb_root):
 
     return output_rst
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser(usage="Export examples to rst")
     parser.add_argument("rst_dir", help="Directory where rst files are generated")
     parser.add_argument("otb_root", help="OTB repository root")
@@ -130,3 +131,6 @@ if __name__ == "__main__":
         os.makedirs(join(args.rst_dir, "C++", "Examples", tag), exist_ok=True)
         with open(join(args.rst_dir, "C++", "Examples", tag, root + ".rst"), "w") as output_file:
             output_file.write(render_example(filename, args.otb_root))
+
+if __name__ == "__main__":
+    main()

Documentation/Cookbook/Scripts/otbGenerateWrappersRstDoc.py

@@ -360,7 +360,7 @@ def multireplace(string, replacements):
 def make_links(text, allapps):
     "Replace name of applications by internal rst links"
 
-    rep = {appname: ":ref:`{}`".format("app-" + appname) for appname in allapps}
+    rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
     return multireplace(text, rep)
 
 def render_application(appname, allapps):
@@ -374,7 +374,7 @@ def render_application(appname, allapps):
     application_documentation_warnings(app)
 
     output = template_application.format(
-        label="app-" + appname,
+        label=appname,
         heading=rst_section(app.GetName(), '='),
         description=app.GetDescription(),
         longdescription=make_links(app.GetDocLongDescription(), allapps),

Documentation/Cookbook/rst/C++.rst

@@ -4,5 +4,14 @@ C++ API
 =======
 
 .. toctree::
+    :maxdepth: 2
 
+    C++/SystemOverview.rst
+    C++/Tutorial.rst
+    C++/Iterators.rst
+    C++/Filters.rst
+    C++/StreamingAndThreading.rst
+    C++/PersistentFilters.rst
+    C++/WriteAnApplication.rst
+    C++/AddingNewModules.rst
     C++/Examples.rst

Documentation/Cookbook/rst/C++/PersistentFilters.rst

+Persistent filters
+==================
+
+Introduction
+------------
+
+As presented in chapter :ref:`StreamingAndThreading`, OTB has two main mechanisms
+to handle large data: streaming allows to process image piece-wise, and
+multi-threading allows to process concurrently several pieces of one streaming
+block. Using these concepts, one can easily write pixel-wise or
+neighborhood-based filters and insert them into a pipeline which will be
+scalable with respect to the input image size.
+
+Yet, sometimes we need to compute global features on the whole image.
+One example is to determine image mean and variance of the input image
+in order to produce a centered and reduced image. The operation of
+centering and reducing each pixel is fully compliant with streaming and
+threading, but one has to first estimate the mean and variance of the
+image. This first step requires to walk the whole image once, and
+traditional streaming and multi-threading based filter architecture is
+of no help here.
+
+This is because there is a fundamental difference between these two
+operations: one supports streaming, and the other needs to perform
+streaming. In fact we would like to stream the whole image piece by
+piece through some filter that will collect and keep mean and variance
+cumulants, and then synthetize theses cumulants to compute the final
+mean and variance once the full image as been streamed. Each stream
+would also benefit from parallel processing. This is exactly what
+persistent filters are for.
+
+Architecture
+------------
+
+There are two main objects in the persistent filters framework. The
+first is the :doxygen:`PersistentImageFilter`, the second is the
+:doxygen:`PersistentFilterStreamingDecorator`.
+
+The persistent filter class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :doxygen:`PersistentImageFilter` class is a regular
+:doxygen-itk:`ImageToImageFilter`, with two additional pure virtual
+methods: the ``Synthetize()`` and the ``Reset()`` methods.
+
+Imagine that the ``GenerateData()`` or ``ThreadedGenerateData()``
+progressively computes some global feature of the whole image, using
+some member of the class to store intermediate results. The
+``Synthetize()`` is an additional method which is designed to be called
+one the whole image has been processed, in order to compute the final
+results from the intermediate results. The ``Reset()`` method is
+designed to allow the reset of the intermediate results members so as to
+start a fresh processing.
+
+Any sub-class of the :doxygen:`PersistentImageFilter` can be used as a
+regular :doxygen-itk:`ImageToImageFilter` (provided that both
+``Synthetize()`` and ``Reset()`` have been implemented, but the real
+interest of these filters is to be used with the streaming decorator
+class presented in the next section.
+
+The streaming decorator class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :doxygen:`PersistentFilterStreamingDecorator` is a class designed to
+be templated with subclasses of the :doxygen:`PersistentImageFilter`. It
+provides the mechanism to stream the whole image through the templated
+filter, using a third class called
+:doxygen:`StreamingImageVirtualWriter`. When the ``Update()`` method is
+called on a :doxygen:`PersistentFilterStreamingDecorator`, a pipeline
+plugging the templated subclass of the :doxygen:`PersistentImageFilter`
+to an instance of :doxygen:`StreamingImageVirtualWriter` is created. The
+latter is then updated, and acts like a regular
+:doxygen:`ImageFileWriter` but it does not actually write anything to
+the disk : streaming pieces are requested and immediately discarded. The
+:doxygen:`PersistentFilterStreamingDecorator` also calls the ``Reset()``
+method at the beginning and the ``Synthetize()`` method at the end of
+the streaming process. Therefore, it packages the whole mechanism for
+the use of a :doxygen:`PersistentImageFilter`:
+
+#. Call the ``Reset()`` method on the filter so as to reset any
+   temporary results members,
+
+#. Stream the image piece-wise through the filter,
+
+#. Call the ``Synthetize()`` method on the filter so as to compute the
+   final results.
+
+There are some methods that allows to tune the behavior of the
+:doxygen:`StreamingImageVirtualWriter`, allowing to change the image
+splitting methods (tiles or strips) or the size of the streams with
+respect to some target available amount of memory. Please see the class
+documentation for details. The instance of the
+:doxygen:`StreamingImageVirtualWriter` can be retrieved from the
+:doxygen:`PersistentFilterStreamingDecorator` through the
+``GetStreamer()`` method.
+
+Though the internal filter of the
+:doxygen:`PersistentFilterStreamingDecorator` can be accessed through
+the ``GetFilter()`` method, the class is often derived to package the
+streaming-decorated filter and wrap the parameters setters and getters.
+
+An end-to-end example
+---------------------
+
+This is an end-to-end example to compute the mean over a full image,
+using a streaming and threading-enabled filter. Please note that only
+specific details are explained here. For more general information on how
+to write a filter, please refer to section :ref:`Filters`.
+
+First step: writing a persistent filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first step is to write a persistent mean image filter. We need to
+include the appropriate header :
+
+.. code-block:: cpp
+
+    #include "otbPersistentImageFilter.h"
+
+Then, we declare the class prototype as follows:
+
+.. code-block:: cpp
+
+    template<class TInputImage>
+    class ITK_EXPORT PersistentMeanImageFilter :
+      public PersistentImageFilter<TInputImage, TInputImage>
+
+Since the output image will only be used for streaming purpose, we do
+not need to declare different input and output template types.
+
+In the *private* section of the class, we will declare a member which
+will be used to store temporary results, and a member which will be used
+to store the final result.
+
+.. code-block:: cpp
+
+    private:
+      // Temporary results container
+      std::vector<PixelType> m_TemporarySums;
+
+      // Final result member
+      double m_Mean;
+
+Next, we will write the ``Reset()`` method implementation in the
+*protected* section of the class. Proper allocation of the temporary
+results container with respect to the number of threads is handled here.
+
+.. code-block:: cpp
+
+    protected:
+      virtual void Reset()
+      {
+        // Retrieve the number of threads
+        unsigned int numberOfThreads = this->GetNumberOfThreads();
+
+        // Reset the temporary results container
+        m_TemporarySums = std::vector<PixelType>(numberOfThreads, itk::NumericTraits<PixelType>::Zero);
+
+        // Reset the final result
+        m_Mean = 0.;
+      }
+
+Now, we need to write the ``ThreadedGenerateData()`` methods (also in
+the *protected* section), were temporary results will be computed for
+each piece of stream.
+
+.. code-block:: cpp
+
+    virtual void ThreadedGenerateData(const RegionType&
+                                      outputRegionForThread,
+                                      itk::ThreadIdType threadId)
+    {
+      // Enable progress reporting
+      itk::ProgressReporter(this,threadId,outputRegionForThread.GetNumberOfPixels());
+
+      // Retrieve the input pointer
+      InputImagePointer inputPtr = const_cast<TInputImage *>(this->GetInput());
+
+      // Declare an iterator on the region
+      itk::ImageRegionConstIteratorWithIndex<TInputImage> it(inputPtr,
+      outputRegionForThread);
+
+      // Walk the region of the image with the iterator
+      for (it.GoToBegin(); !it.IsAtEnd(); ++it, progress.CompletedPixel())
+      {
+        // Retrieve pixel value
+        const PixelType& value = it.Get();
+
+        // Update temporary results for the current thread
+        m_TemporarySums[threadId]+= value;
+      }
+    }
+
+Last, we need to define the ``Synthetize()`` method (still in the
+*protected* section), which will yield the final results:
+
+.. code-block:: cpp
+
+    virtual void Synthetize()
+    {
+      // For each thread
+      for(unsigned int threadId = 0; threadId <this->GetNumberOfThreads();++threadId)
+      {
+        // Update final result
+        m_Mean+=m_TemporarySums[threadId];
+      }
+
+      // Complete calculus by dividing by the total number of pixels
+      unsigned int nbPixels = this->GetInput()->GetLargestPossibleRegion().GetNumberOfPixels();
+
+      if(nbPixels != 0)
+      {
+        m_Mean /= nbPixels;
+      }
+    }
+
+Second step: Decorating the filter and using it
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Now, to use the filter, one only has to decorate it with the
+:doxygen:`PersistentFilterStreamingDecorator`. First step is to include
+the appropriate header:
+
+.. code-block:: cpp
+
+    #include "otbPersistentMeanImageFilter.h"
+    #include "otbPersistentFilterStreamingDecorator.h"
+
+Then, we decorate the filter with some typedefs:
+
+.. code-block:: cpp
+
+    typedef otb::PersistentMeanImageFilter<ImageType> PersitentMeanFilterType;
+    typedef otb::PersistentFilterStreamingDecorator <PersitentMeanFilterType> StreamingMeanFilterType;
+
+Now, the decorated filter can be used like any standard filter:
+
+.. code-block:: cpp
+
+    StreamingMeanFilterType::Pointer filter = StreamingMeanFilterType::New();
+
+    filter->SetInput(reader->GetOutput());
+    filter->Update();
+
+Third step: one class to rule them all
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is often convenient to avoid the few typedefs of the previous section
+by deriving a new class from the decorated filter:
+
+.. code-block:: cpp
+
+    template<class TInputImage>
+    class ITK_EXPORT StreamingMeanImageFilter :
+      public PersistentFilterStreamingDecorator<PersistentImageFilter<TInputImage, TInputImage>>
+
+This also allows to redefine setters and getters for parameters,
+avoiding to call the ``GetFilter()`` method to set them.

Documentation/Cookbook/rst/C++/StreamingAndThreading.rst

+.. _StreamingAndThreading:
+
+Streaming and Threading
+=======================
+
+Streaming and threading are two
+different things even if they are linked to a certain extent. In OTB:
+
+-  Streaming describes the ability to combine the processing of several
+   portion of a big image and to make the output identical as what you
+   would have got if the whole image was processed at once. Streaming is
+   compulsory when you’re processing gigabyte images.
+
+-  Threading is the ability to process simultaneously different parts of
+   the image. Threading will give you some benefits only if you have a
+   fairly recent processor.
+
+To sum up: streaming is good if you have big images, threading is good
+if you have several processing units.
+
+However, these two properties are not unrelated. Both rely on the filter
+ability to process parts of the image and combine the result, that's what
+the ``ThreadedGenerateData()`` method can do.
+
+Streaming and threading in OTB
+------------------------------
+
+For OTB, streaming is pipeline related while threading is filter
+related. If you build a pipeline where one filter is not streamable, the
+whole pipeline is not streamable: at one point, you would hold the
+entire image in memory. Whereas you will benefit from a threaded filter
+even if the rest of the pipeline is made of non-threadable filters (the
+processing time will be shorter for this particular filter).
+
+Even if you use a non streamed writer, each filter which has a
+``ThreadedGenerateData()`` will split the image into two and send each part
+to one thread and you will notice two calls to the function.
+
+If you have some particular requirement and want to use only one thread,
+you can call the ``SetNumberOfThreads()`` method on each of your filter.
+
+When you are writing your own filter, you have to follow some rules to
+make your filter streamable and threadable. Some details are provided in
+sections :ref:`StreamingLargeData` and :ref:`ThreadedFilterExecution`.
+
+Division strategies
+-------------------
+
+The division of the image occurs generally at the writer level.
+Different strategies are available and can be specified explicitly. In
+OTB, these are referred as *splitter*. Several available splitters are:
+
+-  :doxygen-itk:`ImageRegionSplitter`
+
+-  :doxygen-itk:`ImageRegionMultidimensionalSplitter`
+
+-  :doxygen:`ImageRegionNonUniformMultidimensionalSplitter`
+
+You can add your own strategies based on these examples.
+
+To change the splitting strategy of the writer, you can use the
+following model:
+
+.. code-block:: cpp
+
+    typedef otb::ImageRegionNonUniformMultidimensionalSplitter<3> splitterType;
+    splitterType::Pointer splitter=splitterType::New() ;
+    writer->SetRegionSplitter(splitter);

Documentation/Cookbook/rst/C++/Tutorial.rst

+Building simple OTB code
+========================
+
+Well, that’s it, you’ve just downloaded and installed OTB, lured by the
+promise that you will be able to do everything with it. That’s true, you
+will be able to do everything but - there is always a *but* - some
+effort is required.
+
+OTB uses the very powerful systems of generic programming, many classes
+are already available, some powerful tools are defined to help you with
+recurrent tasks, but it is not an easy world to enter.
+
+These tutorials are designed to help you enter this world and grasp the
+logic behind OTB. Each of these tutorials should not take more than 10
+minutes (typing included) and each is designed to highlight a specific
+point. You may not be concerned by the latest tutorials but it is
+strongly advised to go through the first few which cover the basics
+you’ll use almost everywhere.
+
+Hello world
+-----------
+
+Let’s start by the typical *Hello world* program. We are going to
+compile this C++ program linking to your new OTB.
+
+First, create a new folder to put your new programs (all the examples
+from this tutorial) in and go into this folder.
+
+Since all programs using OTB are handled using the CMake system, we need
+to create a ``CMakeLists.txt`` that will be used by CMake to compile our
+program. An example of this file can be found in the
+``OTB/Examples/Tutorials`` directory. The ``CMakeLists.txt`` will be
+very similar between your projects.
+
+Open the ``CMakeLists.txt`` file and write in the few lines:
+
+.. code-block:: cmake
+
+    project(Tutorials)
+
+    cmake_minimum_required(VERSION 3.1.0)
+
+    find_package(OTB)
+    if(OTB_FOUND)
+      include(${OTB_USE_FILE})
+    else(OTB_FOUND)
+      message(FATAL_ERROR "Cannot build OTB project without OTB.  Please set OTB_DIR.")
+    endif(OTB_FOUND)
+
+    add_executable(HelloWorldOTB HelloWorldOTB.cxx )
+    target_link_libraries(HelloWorldOTB ${OTB_LIBRARIES})
+
+The first line defines the name of your project as it appears in Visual
+Studio (it will have no effect under UNIX or Linux). The second line
+loads a CMake file with a predefined strategy for finding OTB. If
+the strategy for finding OTB fails, CMake will prompt you for the
+directory where OTB is installed in your system. In that case you will
+write this information in the ``OTB_DIR`` variable. The line
+``include(${USE_OTB_FILE})`` loads the ``UseOTB.cmake`` file to set all
+the configuration information from OTB.
+
+The line ``add_executable`` defines as its first argument the name of
+the executable that will be produced as result of this project. The
+remaining arguments of ``add_executable`` are the names of the source
+files to be compiled and linked. Finally, the ``target_link_libraries``
+line specifies which OTB libraries will be linked against this project.
+
+:ref:`HelloWorldOTB.cxx`
+
+Once the file is written, run ``ccmake`` on the current directory (that
+is ``ccmake ./`` under Linux/Unix). If OTB is on a non standard place,
+you will have to tell CMake where it is. Once your done with CMake (you
+shouldn’t have to do it anymore) run ``make``.
+
+You finally have your program. When you run it, you will have the *OTB
+Hello World !* printed.
+
+Ok, well done! You’ve just compiled and executed your first OTB program.
+Actually, using OTB for that is not very useful, and we doubt that you
+downloaded OTB only to do that. It’s time to move on to a more advanced
+level.
+
+Pipeline basics: read and write
+-------------------------------
+
+OTB is designed to read images, process them and write them to disk or
+view the result. In this tutorial, we are going to see how to read and
+write images and the basics of the pipeline system.
+
+First, let’s add the following lines at the end of the
+``CMakeLists.txt`` file:
+
+.. code-block:: cmake
+
+    add_executable(Pipeline Pipeline.cxx )
+    target_link_libraries(Pipeline ${OTB_LIBRARIES})
+
+Now, create a ``Pipeline.cxx`` file: :ref:`Pipeline.cxx`.
+
+Once this file is written you just have to run ``make``. The ``ccmake``
+call is not required anymore.
+
+Get one image from the ``OTB-Data/Examples`` directory from the OTB-Data
+repository. You can get it either by cloning the OTB data repository
+(``git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb-data.git``),
+but that might be quite long as this also gets the data to run the
+tests. Alternatively, you can get it from
+http://www.orfeo-toolbox.org/packages/OTB-Data-Examples.tgz. Take for
+example get ``QB_Suburb.png``.
+
+Now, run your new program as ``Pipeline QB_Suburb.png output.png``. You
+obtain the file ``output.png`` which is the same image as
+``QB_Suburb.png``. When you triggered the ``Update()`` method, OTB
+opened the original image and wrote it back under another name.
+
+Well…that’s nice but a bit complicated for a copy program!
+
+Wait a minute! We didn’t specify the file format anywhere! Let’s try
+``Pipeline QB_Suburb.png output.jpg``. And voila! The output image is a
+jpeg file.
+
+That’s starting to be a bit more interesting: this is not just a program
+to copy image files, but also to convert between image formats.
+
+You have just experienced the pipeline structure which executes the
+filters only when needed and the automatic image format detection.
+
+Now it’s time to do some processing in between.
+
+Filtering pipeline
+------------------
+
+We are now going to insert a simple filter to do some processing between
+the reader and the writer.
+
+Let’s first add the 2 following lines to the ``CMakeLists.txt`` file:
+
+.. code-block:: cmake
+
+    add_executable(FilteringPipeline FilteringPipeline.cxx )
+    target_link_libraries(FilteringPipeline ${OTB_LIBRARIES})
+
+See example :ref:`FilteringPipeline.cxx`
+
+Compile with ``make`` and execute as
+``FilteringPipeline QB_Suburb.png output.png``.
+
+You have the filtered version of your image in the ``output.png`` file.
+
+Now, you can practice a bit and try to replace the filter by one of the
+150+ filters which inherit from the :doxygen-itk:`ImageToImageFilter`
+class. You will definitely find some useful filters here!
+
+Handling types: scaling output
+------------------------------
+
+If you tried some other filter in the previous example, you may have
+noticed that in some cases, it does not make sense to save the output
+directly as an integer. This is the case if you tried the
+:doxygen-itk:`CannyEdgeDetectionImageFilter`. If you tried to use it
+directly in the previous example, you will have some warning about
+converting to unsigned char from double.
+
+The output of the Canny edge detection is a floating point number. A
+simple solution would be to used double as the pixel type.
+Unfortunately, most image formats use integer typed and you should
+convert the result to an integer image if you still want to visualize
+your images with your usual viewer (we will see in a tutorial later how
+you can avoid that using the built-in viewer).
+
+To realize this conversion, we will use the
+:doxygen-itk:`RescaleIntensityImageFilter`.
+
+Add the two lines to the ``CMakeLists.txt`` file:
+
+.. code-block:: cmake
+
+    add_executable(ScalingPipeline ScalingPipeline.cxx )
+    target_link_libraries(ScalingPipeline ${OTB_LIBRARIES})
+
+See example :ref:`ScalingPipeline.cxx`
+
+As you should be getting used to it by now, compile with ``make`` and
+execute as ``ScalingPipeline QB_Suburb.png output.png``.
+
+You have the filtered version of your image in the ``output.png`` file.
+
+Working with multispectral or color images
+------------------------------------------
+
+So far, as you may have noticed, we have been working with grey level
+images, i.e. with only one spectral band. If you tried to process a
+color image with some of the previous examples you have probably
+obtained a deceiving grey result.
+
+Often, satellite images combine several spectral band to help the
+identification of materials: this is called multispectral imagery. In
+this tutorial, we are going to explore some of the mechanisms used by
+OTB to process multispectral images.
+
+Add the following lines in the ``CMakeLists.txt`` file:
+
+.. code-block:: cmake