Commit fe3ca431 authored by Cédric Traizet's avatar Cédric Traizet

Merge branch 'develop' into superbuild_data

parents 906e7231 d907604a
Pipeline #1462 passed with stage
in 8 minutes and 3 seconds
......@@ -24,10 +24,10 @@ before_script:
# Provision efficiently the local LFS cache before checkout
- git lfs fetch origin $CI_COMMIT_SHA
# Checkout the expected branch
- git checkout $CI_COMMIT_REF_NAME
- git checkout -f -q $CI_COMMIT_SHA
- precheck
......@@ -64,25 +64,29 @@ if __name__ == "__main__":
sha1 = env['CI_COMMIT_SHA']
# are we in a merge_request pipeline ?
if 'CI_MERGE_REQUEST_IID' in env.keys():
if not CheckEnvParameters(['K8S_SECRET_API_TOKEN']):
print("Make sure you have set a valid acces token for Gitlab API")
print("The K8S_SECRET_API_TOKEN environment variable should be set in 'Settings -> CI/CD -> Variables'")
mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_TWIN_PIPELINE'])
if not CheckEnvParameters(['CI_PROJECT_ID','CI_PIPELINE_ID']):
mrInfo = GitlabRequest('merge_requests/'+env['CI_MERGE_REQUEST_IID'],token=env['K8S_SECRET_API_TOKEN'])
wip_regex = re.compile("^[Ww][Ii][Pp]:")
# is it a "WIP" merge request ?
# Yes: cancel the current pipeline
print("Cancel current pipeline "+env['CI_PIPELINE_ID'])
GitlabRequest('pipelines/'+env['CI_PIPELINE_ID']+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_API_TOKEN'])
print("Error: this pipeline should have been canceled")
# No: cancel any previous "normal" pipeline on the same SHA1
jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
jres = GitlabRequest('pipelines?sha='+sha1, project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_API_TOKEN'])
for item in jres:
if item["id"] < int(env['CI_PIPELINE_ID']) and item["status"] == "running":
print("Cancel pipeline "+str(item["id"]))
jres2 = GitlabRequest('pipelines/'+str(item["id"])+'/cancel', data={}, \
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_TWIN_PIPELINE'])
project=env['CI_PROJECT_ID'], token=env['K8S_SECRET_API_TOKEN'])
......@@ -55,8 +55,7 @@ OTB_USE_SSE_FLAGS:BOOL=ON
set (otb_wrap_option
set (otb_data_option
......@@ -165,9 +165,6 @@ endif()
set(OTB_INSTALL_DATA_DIR "share/otb")
......@@ -533,7 +530,6 @@ foreach(otb_option
......@@ -5,7 +5,6 @@
<descr>This application helps developers to test parameters types</descr>
<longdescr>The purpose of this application is to test parameters types.</longdescr>
......@@ -25,7 +25,7 @@ import argparse
import re
import otbApplication
from otbApplication import ParameterType_Bool, ParameterType_Int, ParameterType_Radius, ParameterType_RAM, ParameterType_Float, ParameterType_String, ParameterType_StringList, ParameterType_InputFilename, ParameterType_OutputFilename, ParameterType_InputImage, ParameterType_OutputImage, ParameterType_InputVectorData, ParameterType_OutputVectorData, ParameterType_Directory, ParameterType_Choice, ParameterType_InputImageList, ParameterType_InputVectorDataList, ParameterType_InputFilenameList, ParameterType_InputProcessXML, ParameterType_OutputProcessXML, ParameterType_ListView, ParameterType_Group
from otbApplication import ParameterType_Bool, ParameterType_Int, ParameterType_Radius, ParameterType_RAM, ParameterType_Float, ParameterType_String, ParameterType_StringList, ParameterType_InputFilename, ParameterType_OutputFilename, ParameterType_InputImage, ParameterType_OutputImage, ParameterType_InputVectorData, ParameterType_OutputVectorData, ParameterType_Directory, ParameterType_Choice, ParameterType_InputImageList, ParameterType_InputVectorDataList, ParameterType_InputFilenameList, ParameterType_ListView, ParameterType_Group
from otb_warnings import application_documentation_warnings
......@@ -202,7 +202,6 @@ def rst_parameter_value(app, key):
values.update({ParameterType_InputImageList: "image1 image2..."})
values.update({ParameterType_InputVectorDataList: "vectorfile1 vectorfile2..."})
values.update({ParameterType_InputFilenameList: "filename1 filename2..."})
values.update(dict.fromkeys([ParameterType_InputProcessXML, ParameterType_OutputProcessXML], "filename.xml"))
if type in values:
return values[type]
......@@ -356,10 +355,17 @@ def make_links(text, allapps):
rep = {appname: ":ref:`{}`".format(appname) for appname in allapps}
return multireplace(text, rep)
def render_deprecation_string(app):
if app.IsDeprecated():
return "This application is deprecated and will be removed in a future release."
return ""
def render_application(appname, allapps):
"Render app to rst"
app = otbApplication.Registry.CreateApplication(appname)
# Create the application without logger to avoid the deprecation warning log
app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
# TODO: remove this when bug 440 is fixed
......@@ -368,6 +374,7 @@ def render_application(appname, allapps):
output = template_application.format(
heading=rst_section(app.GetName(), '='),
longdescription=make_links(app.GetDocLongDescription(), allapps),
......@@ -381,8 +388,9 @@ def render_application(appname, allapps):
return output
def GetApplicationTags(appname):
app = otbApplication.Registry.CreateApplication(appname)
return app.GetDocTags()
# Create the application without logger to avoid the deprecation warning log
app = otbApplication.Registry.CreateApplicationWithoutLogger(appname)
return app.GetDocTags()
def GenerateRstForApplications(rst_dir):
"Generate .rst files for all applications"
......@@ -404,9 +412,12 @@ def GenerateRstForApplications(rst_dir):
for appName in appNames:
# Get application first tag
tags = GetApplicationTags(appName)
tags = list(GetApplicationTags(appName))
if "Deprecated" in tags:
if not tags or len(tags) == 0:
raise RuntimeError("No tags for application: " + appName)
tag = tags[0]
tag_ = tag.replace(" ", "_")
......@@ -70,8 +70,6 @@ Variables and their descriptions:
| im1PhyX and im1PhyY | spacing of first input in X and Y directions | Scalar |
In addition, we also have the generic variables idxX and idxY that
represent the indices of the current pixel (scalars).
......@@ -83,8 +81,6 @@ For instance, the following formula (addition of two pixels)
.. math:: im1+im2
is correct only if the two first inputs have the same number of bands.
In addition, the following formula is not consistent even if im1
represents a pixel of an image made of only one band:
......@@ -165,7 +161,6 @@ For instance, im1b3N3x5 represents the following neighbourhood:
| . | . | . |
Fundamentally, a neighbourhood is represented as a matrix inside the
muParserX framework; so the remark about mathematically well-defined
......@@ -188,7 +183,7 @@ BandMathImageFilterX. These ones can be divided into two categories.
Concerning the last category, here is a list of implemented operators or
functions (they are all implemented in otbParserXPlugins.h/.cxx files
**Operators div and dv** The first operator allows the definition of an
element-wise division of two vectors (and even matrices), provided that
......@@ -230,7 +225,7 @@ produces a vector of 4 components made of band 1, band 2, band 1 and
band 1 values from the first input. Note that curly brackets must be
used in order to select the desired band indices.
** Function dotpr ** This function allows the dot product between two
**Function dotpr** This function allows the dot product between two
vectors or matrices (actually in our case, a kernel and a neighbourhood
of pixels):
......@@ -278,7 +273,7 @@ instance:
.. math:: maj(im1b1N3x3,im1b2N3x3)
**Function vmin and vmax** These functions calculate the min or
max value of a given vector or neighborhood (only one input). For
max value of a given vector or neighborhood (only one input, the output is a 1x1 matrix). For
.. math:: (vmax(im3b1N3x5)+vmin(im3b1N3x5)) ~ div ~ \{2.0\}
......@@ -365,8 +360,8 @@ Functions and operators summary:
| vsqrt | adapation of an existing function to vectors: one input |
| vect2scal | one dimensional vector to scalar |
.. _API:
......@@ -454,11 +449,6 @@ defines the kernel1, whose elements are given as follows:
| 1,3 | 1,4 | 1,5 |
Definition of kernel1.
/** Set a constant */
......@@ -158,9 +158,6 @@ parameters:
- ``ParameterType_InputVectorDataList`` : parameter storing a list of
input vector data.
- ``ParameterType_InputProcessXML`` : parameter storing an input XML
file name.
- ``ParameterType_OutputFilename`` : parameter storing an output file
......@@ -169,9 +166,6 @@ parameters:
- ``ParameterType_OutputVectorData`` : parameter storing an output
vector data.
- ``ParameterType_OutputProcessXML`` : parameter storing an output XML
file name.
- ``ParameterType_RAM`` : parameter storing the maximum amount of RAM
to be used.
......@@ -123,7 +123,6 @@ example based on the OrthoRectification application:
-opt.rpc <int32> RPC modeling (points per axis) (optional, off by default, default value is 10)
-opt.ram <int32> Available RAM (MB) (optional, off by default, default value is 128)
-opt.gridspacing <float> Resampling grid spacing (optional, off by default, default value is 4)
-inxml <string> Load parameters from XML (optional, off by default)
-progress <boolean> Report progress
-help <string list> Display long help (empty list), or help for given parameters keys
......@@ -145,11 +144,10 @@ Command-line examples are provided in the chapter :ref:`apprefdoc`.
Load and save parameters to XML
Since OTB 3.20, OTB applications parameters can be export/import to/from
an XML file using inxml/outxml parameters. Those parameters are
available in all applications.
An example is worth a thousand words
OTB application parameters can be saved and loaded to
an XML file using the special ``-inxml`` and ``-outxml`` parameters.
Those parameters are available in all applications.
For example:
......@@ -177,17 +175,6 @@ time
In this case it will use as mathematical expression “(im1b1 - im2b1)”
instead of “abs(im1b1 - im2b1)”.
Finally, you can also launch applications directly from the command-line
launcher executable using the inxml parameter without having to declare
the application name. Use in this case:
otbApplicationLauncherCommandLine -inxml saved_applications_parameters.xml
It will retrieve the application name and related parameters from the
input XML file and launch in this case the BandMath applications.
Parallel execution with MPI
......@@ -152,3 +152,15 @@ following repositories with these command-lines: tzotsos
and then add the OTB packages as shown above.
Package is in AUR (Arch User Repository).
You will then need to run (if you use yaourt, you could use trizen instead):
yaourt -S orfeo-toolbox
......@@ -218,11 +218,34 @@ implementation does not break it, for instance by using an internal
writer to write intermediate data. In this case, execution should
still be correct, but some intermediate data will be read or written.
Load and save parameters to XML
As with a the `command line interface` you can save application parameters
to an xml file:
.. code-block:: python
# Save application parameters to XML
app = otb.Registry.CreateApplication('BandMath')
app.SetParameterStringList("il", ["image1.tif", "image2.tif"], True)
app.SetParameterString("out", out, True)
app.SetParameterString("exp", "cos(im1b1)+im2b1*im1b1", True)
And load them later for execution:
.. code-block:: python
# Load application parameters from XML
app = otb.Registry.CreateApplication("BandMath")
Interactions with OTB pipeline
[Since OTB 6.6]
The application framework has been extended in order to provide ways to
interact with the pipelines inside each application. It applies only to
applications that use input or output images. Let's check which
......@@ -22,7 +22,7 @@ The naming convention “im[x]b[y]” designates the yth band of the xth
input image.
The *BandMath* application embeds built-in operators and functions
listed in `muparser documentation <>`_ thus
listed in `muparser documentation <>`_ thus
allowing a vast choice of possible operations.
Images with no-data values
.. _{label}:
......@@ -68,7 +68,6 @@ private:
// \begin{description}
// \item[\code{SetName()}] Name of the application.
// \item[\code{SetDescription()}] Set the short description of the class.
// \item[\code{SetDocName()}] Set long name of the application (that can be displayed \dots).
// \item[\code{SetDocLongDescription()}] This methods is used to describe the class.
// \item[\code{SetDocLimitations()}] Set known limitations (threading, invalid pixel type \dots) or bugs.
// \item[\code{SetDocAuthors()}] Set the application Authors. Author List. Format : "John Doe, Winnie the Pooh" \dots
......@@ -81,7 +80,6 @@ private:
"Pay attention, it includes Latex snippets in order to generate "
"software guide documentation");
"The purpose of this application is "
"to present parameters types,"
......@@ -49,7 +49,6 @@ private:
SetDescription("Change detection by Multivariate Alteration Detector (MAD) algorithm");
// Documentation
SetDocName("Multivariate Alteration Detector");
SetDocLongDescription("This application performs change detection between two multispectral"
" images using the Multivariate Alteration Detector (MAD) [1]"
" algorithm.\n\n"
......@@ -65,6 +65,11 @@ otb_create_application(
SOURCES otbTrainRegression.cxx
NAME TrainVectorRegression
SOURCES otbTrainVectorRegression.cxx
NAME PredictRegression
SOURCES otbPredictRegression.cxx
......@@ -62,7 +62,6 @@ private:
SetDescription("Filters the input labeled image using Majority Voting in a ball shaped neighbordhood");
SetDocName("Classification Map Regularization");
"This application filters the input labeled image (with a maximal class label = 65535) using Majority Voting in a ball shaped neighbordhood."
......@@ -109,7 +109,6 @@ private:
SetDescription("Computes the confusion matrix of a classification");
// Documentation
SetDocName("Confusion matrix Computation");
SetDocLongDescription("This application computes the confusion matrix of a classification map relative to a ground truth dataset. "
"This ground truth can be given as a raster or a vector data. Only reference and produced pixels with values different "
"from NoData are handled in the calculation of the confusion matrix. The confusion matrix is organized the following way: "
......@@ -48,7 +48,6 @@ private:
void DoInit() override
SetDocName("Compute Images second order statistics");
SetDescription("Computes global mean and standard deviation for each band "
"from a set of images and optionally saves the results in an XML file.");
SetDocLongDescription("This application computes a global mean and standard deviation "
......@@ -51,7 +51,6 @@ private:
SetDescription("Compute statistics of the features in a set of OGR Layers");
SetDocLongDescription("Compute statistics (mean and standard deviation) of the features in a set of OGR Layers, and write them in an XML file. This XML file can then be used by the training application.");
SetDocLimitations("Experimental. For now only shapefiles are supported.");
SetDocAuthors("David Youssefi during internship at CNES");
......@@ -97,7 +97,6 @@ private:
SetDescription("Fuses several classifications maps of the same image on the basis of class labels.");
SetDocName("Fusion of Classifications");
SetDocLongDescription("This application allows you to fuse several classification maps and produces a single more robust classification map. "
"Fusion is done either by mean of Majority Voting, or with the Dempster Shafer combination method on class labels.\n\n"
" - MAJORITY VOTING: for each pixel, the class with the highest number of votes is selected.\n"
......@@ -80,7 +80,6 @@ private:
SetDescription("Performs a classification of the input image according to a model file.");
// Documentation
SetDocName("Image Classification");
SetDocLongDescription("This application performs an image classification based on a model file produced by the TrainImagesClassifier application. Pixels of the output image will contain the class labels decided by the classifier (maximal class label = 65535). The input pixels can be optionally centered and reduced according to the statistics file produced by the ComputeImagesStatistics application. An optional input mask can be provided, in which case only input image pixels whose corresponding mask value is greater than 0 will be classified. By default, the remaining of pixels will be given the label 0 in the output image.");
SetDocLimitations("The input image must have the same type, order and number of bands than the images used to produce the statistics file and the SVM model file. If a statistics file was used during training by the TrainImagesClassifier, it is mandatory to use the same statistics file for classification. If an input mask is used, its size must match the input image size.");
......@@ -371,7 +371,6 @@ private:
SetDescription("Unsupervised KMeans image classification");
SetDocName("Unsupervised KMeans image classification");
SetDocLongDescription("Unsupervised KMeans image classification. "
"This is a composite application, using existing training and classification applications. "
"The SharkKMeans model is used.\n\n"
......@@ -63,7 +63,6 @@ private:
SetDescription("Compute sampling rate for an input set of images.");
// Documentation
SetDocName("Multi-image sampling rate estimation");
SetDocLongDescription("The application computes sampling rates for a set of"
" input images. Before calling this application, each pair of image and "
"training vectors has to be analysed with the application "
......@@ -60,7 +60,6 @@ private:
SetDescription("Classify an OGR layer based on a machine learning model and a list of features to consider.");
SetDocLongDescription("This application will apply a trained machine learning model on the selected feature to get a classification of each geometry contained in an OGR layer. The list of feature must match the list used for training. The predicted label is written in the user defined field for each geometry.");
SetDocLimitations("Experimental. Only shapefiles are supported for now.");
SetDocAuthors("David Youssefi during internship at CNES");
......@@ -72,7 +72,6 @@ private:
SetDescription("Computes statistics on a training polygon set.");
// Documentation
SetDocName("Polygon Class Statistics");
SetDocLongDescription("Process a set of geometries intended for training (they should have a field giving the associated "
"class). The geometries are analyzed against a support image to compute statistics:\n\n"
"* Number of samples per class\n"
......@@ -119,7 +119,6 @@ private:
SetDescription("Performs a prediction of the input image according to a regression model file.");
// Documentation
SetDocName("Predict Regression");
SetDocLongDescription("This application predict output values from an input"
" image, based on a regression model file produced by"
" the TrainRegression application. Pixels of the "
......@@ -82,7 +82,6 @@ private:
SetDescription("SOM image classification.");
// Documentation
SetDocName("SOM Classification");
SetDocLongDescription("Unsupervised Self Organizing Map image classification.");
......@@ -57,7 +57,6 @@ private:
SetDescription("Generates synthetic samples from a sample data file.");
// Documentation
SetDocName("Sample Augmentation");
SetDocLongDescription("The application takes a sample data file as "
"generated by the SampleExtraction application and "
"generates synthetic samples to increase the number of "
......@@ -59,7 +59,6 @@ private:
SetDescription("Extracts samples values from an image.");
// Documentation
SetDocName("Sample Extraction");
SetDocLongDescription("The application extracts samples values from an"
"image using positions contained in a vector data file. ");
......@@ -86,7 +86,6 @@ private:
SetDescription("Selects samples from a training vector data set.");
// Documentation
SetDocName("Sample Selection");
"The application selects a set of samples from geometries "
"intended for training (they should have a field giving the associated "
......@@ -41,7 +41,6 @@ public:
SetDescription( "Train a classifier from multiple pairs of images and training vector data." );
// Documentation
SetDocName( "Train a classifier from multiple images" );
"Train a classifier from multiple pairs of images and training vector data. "
"Samples are composed of pixel values in each band optionally centered and reduced using an XML statistics file produced by "
......@@ -105,7 +105,6 @@ void DoInit() override
"Train a classifier from multiple images to perform regression.");
// Documentation
SetDocName("Train a regression model");
"This application trains a classifier from multiple input images or a csv "
"file, in order to perform regression. Predictors are composed of pixel "
......@@ -272,8 +271,7 @@ void ParseCSVPredictors(std::string path, ListSampleType* outputList)
for (unsigned int i=0 ; i<nbCols ; ++i)
iss >> elem[i];
elem[i] = std::stod(words[i]);
......@@ -29,11 +29,11 @@ namespace otb
namespace Wrapper
class TrainVectorClassifier : public TrainVectorBase
class TrainVectorClassifier : public TrainVectorBase<float, int>
typedef TrainVectorClassifier Self;
typedef TrainVectorBase Superclass;
typedef TrainVectorBase<float, int> Superclass;
typedef itk::SmartPointer<Self> Pointer;
typedef itk::SmartPointer<const Self> ConstPointer;
itkNewMacro( Self )
......@@ -60,20 +60,26 @@ protected:
SetDescription( "Train a classifier based on labeled geometries and a "
"list of features to consider." );
SetDocName( "Train Vector Classifier" );
SetDocLongDescription( "This application trains a classifier based on "
"labeled geometries and a list of features to consider for "
"classification.\nThis application is based on LibSVM, OpenCV Machine "
"Learning (2.3.1 and later), and Shark ML The output of this application "
"is a text model file, whose format corresponds to the ML model type "
"chosen. There is no image nor vector data output.");
SetDocAuthors( "OTB Team" );
SetDocSeeAlso( " " );
// Add a new parameter to compute confusion matrix / contingency table
this->AddParameter(ParameterType_OutputFilename, "io.confmatout", "Output confusion matrix or contingency table");
"Output file containing the confusion matrix or contingency table (.csv format)."
"The contingency table is output when we unsupervised algorithms is used otherwise the confusion matrix is output.");
void DoUpdateParameters() override
* Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
* This file is part of Orfeo Toolbox
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
#include "otbTrainVectorBase.h"
namespace otb
namespace Wrapper