Commit 7fd82ff2 authored by Guillaume Pasero's avatar Guillaume Pasero

Merge branch 'switch-app-chain' into 'develop'

Switch for  application connections

See merge request !327
parents e2b20ee6 f5564eee
Pipeline #2181 passed with stages
in 90 minutes and 49 seconds
......@@ -218,6 +218,68 @@ implementation does not break it, for instance by using an internal
writer to write intermediate data. In this case, execution should
still be correct, but some intermediate data will be read or written.
Mixed in-memory / on-disk connection
------------------------------------
As an extension to the connection of OTB Applications (described in previous
section), a mixed mode is also available to easily switch between:
- **in-memory**: if you want to avoid unecessary I/O between applications
- **on-disk**: if you want intermediate outputs on disk
This mixed mode is based on the ``Application::ConnectImage()`` function. This
is how you use it:
.. code-block:: python
# for in-memory mode
app1 = otb.Registry.CreateApplication("Smoothing")
app2 = otb.Registry.CreateApplication("Smoothing")
app1.IN = input_image
app2.ConnectImage("in", app1, "out")
app2.OUT = output_image
app2.ExecuteAndWriteOutput()
Comparing with the standard in-memory connection, you can notice that:
- the syntax to do the connection is simpler
- you don't need to call ``Execute()`` on upstream applications anymore,
this is done recursively by calling ``ExecuteAndWriteOutput()`` on the latest
application
The on-disk version of this code is very similar:
.. code-block:: python
# for on-disk mode
app1 = otb.Registry.CreateApplication("Smoothing")
app2 = otb.Registry.CreateApplication("Smoothing")
app1.IN = input_image
app1.OUT = temp_image
app2.ConnectImage("in", app1, "out")
app2.OUT = output_image
app2.PropagateConnectMode(False)
app2.ExecuteAndWriteOutput()
The function ``PropagateConnectMode()`` is applied recursively in the upstream
applications. It allows to change between the 2 modes:
- ``True`` : means in-memory mode (this is the default)
- ``False`` : means on-disk mode
When you want to use the on-disk mode, you have to specify the path to
the intermediate image in the **output** image parameter of the upstream
application (``app1.OUT`` in the previous example). If this path is empty, the
in-memory mode is used as fallback.
This feature also works for ``InputImageList``. Calling the function
``ConnectImage("il", app1, "out")``, with ``il`` being an input image list, will
append a new image to the list, and connect it to output parameter ``out``.
Load and save parameters to XML
-------------------------------
......
......@@ -121,6 +121,12 @@ public:
*/
int Execute();
/** write all of the output to disk
* if they have an associated filename.
* This is a helper function for wrappers without pipeline support.
*/
void WriteOutput();
/** Run the application, then write all of the output to disk
* if they have an associated filename.
* This is a helper function for wrappers without pipeline support.
......@@ -129,6 +135,12 @@ public:
*/
int ExecuteAndWriteOutput();
/** Connect input image to an output image in app */
bool ConnectImage(std::string in, Application* app, std::string out);
/** Propagate the connection mode : */
void PropagateConnectMode(bool isMem);
/** Request the application to stop its processing */
void Stop();
......@@ -772,6 +784,8 @@ public:
*/
void FreeRessources();
bool IsExecuteDone();
protected:
/** Constructor */
Application();
......@@ -894,6 +908,9 @@ private:
/** Flag is true when executing DoInit, DoUpdateParameters or DoExecute */
bool m_IsInPrivateDo;
/** Flag to check if Execute has already been called */
bool m_ExecuteDone;
/**
* Declare the class
* - Wrapper::MapProjectionParametersHandler
......
......@@ -55,10 +55,31 @@ public:
/** RTTI support */
itkTypeMacro(InputImageParameter, Parameter);
typedef struct
{
itk::Object::Pointer app;
std::string key;
bool isMem;
} Connector;
/** Set value from filename */
bool SetFromFileName( const std::string & filename );
itkGetConstReferenceMacro( FileName, std::string );
void SetConnection(Connector c)
{
m_Connection = c;
}
const Connector & GetConnection() const
{
return m_Connection;
}
void SetConnectionMode(bool isMem)
{
m_Connection.isMem = isMem;
}
/** Get input-image as ImageBaseType. */
ImageBaseType const* GetImage() const;
......@@ -152,6 +173,8 @@ private:
/** flag : are we using a filename or an image pointer as an input */
bool m_UseFilename;
Connector m_Connection;
}; // End class InputImage Parameter
} // End namespace Wrapper
......
......@@ -86,6 +86,8 @@ public:
/** */
void Insert( const std::string &, std::size_t = -1 ) override;
void InsertElement(typename T::Pointer, std::size_t = -1);
/** Set one specific stored filename. */
void SetNthFileName( std::size_t, const std::string & ) override;
......@@ -117,6 +119,8 @@ public:
/** */
void Swap( std::size_t, std::size_t ) override;
typename T::Pointer GetNthElement(std::size_t);
std::vector<std::string> ToStringList() const override;
void FromStringList(const std::vector<std::string>& value) override;
std::string ToString() const override;
......
......@@ -159,8 +159,17 @@ ParameterList< T >
p->FromString(filename);
m_Parameters.insert( m_Parameters.begin() + index, p );
InsertElement(p, index);
}
/*****************************************************************************/
template< typename T >
void
ParameterList< T >
::InsertElement(typename T::Pointer p, std::size_t index)
{
m_Parameters.insert( m_Parameters.begin() + index, p );
assert( !m_Parameters.back().IsNull() );
SetActive( true );
......@@ -443,9 +452,9 @@ ParameterList< T >
{
assert( data!=nullptr );
typename T::Pointer p;
typename T::Pointer p( T::New() );
return From( p, data, set, description );
return FromData( p, data, set, description );
}
/*****************************************************************************/
......@@ -460,8 +469,6 @@ ParameterList< T >
{
assert( data!=nullptr );
parameter = T::New();
set( parameter, data );
parameter->SetDescription( description );
......@@ -497,6 +504,14 @@ std::string ParameterList<T>::ToString() const
return oss.str();
}
template< typename T >
typename T::Pointer
ParameterList< T >
::GetNthElement(std::size_t i)
{
return m_Parameters[i];
}
} // End namespace Wrapper
......
......@@ -38,13 +38,16 @@
#include "otbWrapperBoolParameter.h"
#include "otbWrapperAddProcessToWatchEvent.h"
#include "otbExtendedFilenameToWriterOptions.h"
#include "otbCast.h"
#include "otbMacro.h"
#include "otbWrapperTypes.h"
#include <exception>
#include "itkMacro.h"
#include <stack>
#include <set>
#include <unordered_set>
namespace otb
{
......@@ -334,7 +337,8 @@ Application::Application()
m_DocSeeAlso(""),
m_DocTags(),
m_Doclink(""),
m_IsInPrivateDo(false)
m_IsInPrivateDo(false),
m_ExecuteDone(false)
{
// Don't call Init from the constructor, since it calls a virtual method !
m_Logger->SetName("Application.logger");
......@@ -467,6 +471,8 @@ void Application::UpdateParameters()
m_IsInPrivateDo = true;
this->DoUpdateParameters();
m_IsInPrivateDo = false;
// reset the flag m_ExecuteDone
m_ExecuteDone = false;
}
void Application::AfterExecuteAndWriteOutputs()
......@@ -675,11 +681,119 @@ void Application::FreeRessources()
int Application::Execute()
{
//----------- Recursive part -------------------------------------------------
std::vector<std::string> paramList = GetParametersKeys(true);
int status=0;
std::unordered_set<Application*> targetApps;
for (std::vector<std::string>::const_iterator it = paramList.begin(); it != paramList.end(); ++it)
{
std::string key = *it;
Parameter* param = GetParameterByKey(key);
InputImageParameter* imgParam = dynamic_cast<InputImageParameter*>(param);
if(imgParam)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgParam->GetConnection().app);
if(targetApp.IsNotNull() && !targetApp->IsExecuteDone())
{
targetApps.insert(targetApp);
}
}
else
{
InputImageListParameter* imgListParam = dynamic_cast<InputImageListParameter*>(param);
if (imgListParam)
{
for (unsigned int i=0 ; i<imgListParam->Size(); i++)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgListParam->GetNthElement(i)->GetConnection().app);
if(targetApp.IsNotNull() && !targetApp->IsExecuteDone())
{
targetApps.insert(targetApp);
}
}
}
}
}
for (auto &app : targetApps)
{
// Call target Execute()
status = status | app->Execute();
}
for (std::vector<std::string>::const_iterator it = paramList.begin(); it != paramList.end(); ++it)
{
std::string key = *it;
Parameter* param = GetParameterByKey(key);
InputImageParameter* imgParam = dynamic_cast<InputImageParameter*>(param);
if(imgParam)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgParam->GetConnection().app);
if(targetApp.IsNotNull())
{
std::string outKey = imgParam->GetConnection().key;
if(imgParam->GetConnection().isMem || !targetApp->HasValue(outKey))
{
// memory connection
SetParameterInputImage(key,
targetApp->GetParameterOutputImage(outKey));
targetApp->DisableParameter(outKey);
}
else
{
// set input string based on out image (and strip any extended filename)
otb::ExtendedFilenameToWriterOptions::Pointer fnHelper = otb::ExtendedFilenameToWriterOptions::New();
fnHelper->SetExtendedFileName(targetApp->GetParameterString(outKey));
SetParameterString(key, fnHelper->GetSimpleFileName() );
targetApp->EnableParameter(outKey);
}
}
}
else
{
InputImageListParameter* imgListParam = dynamic_cast<InputImageListParameter*>(param);
if (imgListParam)
{
for (unsigned int i=0 ; i<imgListParam->Size() ; i++)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgListParam->GetNthElement(i)->GetConnection().app);
if(targetApp.IsNotNull())
{
std::string outKey = imgListParam->GetNthElement(i)->GetConnection().key;
if(imgListParam->GetNthElement(i)->GetConnection().isMem ||
!targetApp->HasValue(outKey))
{
// memory connection
SetNthParameterInputImageList(key,i,
targetApp->GetParameterOutputImage(outKey));
targetApp->DisableParameter(outKey);
}
else
{
// set input string based on out image (and strip any extended filename)
otb::ExtendedFilenameToWriterOptions::Pointer fnHelper = otb::ExtendedFilenameToWriterOptions::New();
fnHelper->SetExtendedFileName(targetApp->GetParameterString(outKey));
SetNthParameterStringList(key, i, fnHelper->GetSimpleFileName());
targetApp->EnableParameter(outKey);
}
}
}
}
}
}
if (status != 0)
{
return status;
}
for (auto &app : targetApps)
{
app->WriteOutput();
}
//------------------------------------------------------------
this->UpdateParameters();
// before execute we set the seed of mersenne twister
std::vector<std::string> paramList = GetParametersKeys(true);
bool UseSpecificSeed = false;
for (std::vector<std::string>::const_iterator it = paramList.begin(); it != paramList.end(); ++it)
......@@ -705,6 +819,7 @@ int Application::Execute()
m_IsInPrivateDo = true;
this->DoExecute();
m_IsInPrivateDo = false;
m_ExecuteDone = true;
// Ensure that all output image parameter have called UpdateOutputInformation()
for (auto it = paramList.begin(); it != paramList.end(); ++it)
......@@ -725,84 +840,91 @@ int Application::Execute()
return 0;
}
int Application::ExecuteAndWriteOutput()
void Application::WriteOutput()
{
m_Chrono.Restart();
m_Logger->LogSetupInformation();
int status = this->Execute();
if (status == 0)
std::vector<std::string> paramList = GetParametersKeys(true);
// First Get the value of the available memory to use with the
// writer if a RAMParameter is set
bool useRAM = false;
unsigned int ram = 0;
for (std::vector<std::string>::const_iterator it = paramList.begin();
it != paramList.end();
++it)
{
std::vector<std::string> paramList = GetParametersKeys(true);
// First Get the value of the available memory to use with the
// writer if a RAMParameter is set
bool useRAM = false;
unsigned int ram = 0;
for (std::vector<std::string>::const_iterator it = paramList.begin();
it != paramList.end();
++it)
{
std::string key = *it;
std::string key = *it;
if (GetParameterType(key) == ParameterType_RAM
&& IsParameterEnabled(key))
{
Parameter* param = GetParameterByKey(key);
RAMParameter* ramParam = dynamic_cast<RAMParameter*>(param);
if(ramParam!=nullptr)
{
ram = ramParam->GetValue();
useRAM = true;
}
}
if (GetParameterType(key) == ParameterType_RAM
&& IsParameterEnabled(key))
{
Parameter* param = GetParameterByKey(key);
RAMParameter* ramParam = dynamic_cast<RAMParameter*>(param);
if(ramParam!=nullptr)
{
ram = ramParam->GetValue();
useRAM = true;
}
}
}
for (std::vector<std::string>::const_iterator it = paramList.begin();
it != paramList.end();
++it)
for (std::vector<std::string>::const_iterator it = paramList.begin();
it != paramList.end();
++it)
{
std::string key = *it;
if (GetParameterType(key) == ParameterType_OutputImage
&& IsParameterEnabled(key) && HasValue(key) )
{
Parameter* param = GetParameterByKey(key);
OutputImageParameter* outputParam = dynamic_cast<OutputImageParameter*>(param);
if(outputParam!=nullptr)
{
std::string key = *it;
if (GetParameterType(key) == ParameterType_OutputImage
&& IsParameterEnabled(key) && HasValue(key) )
std::string checkReturn = outputParam->CheckFileName(true);
if (!checkReturn.empty())
{
Parameter* param = GetParameterByKey(key);
OutputImageParameter* outputParam = dynamic_cast<OutputImageParameter*>(param);
if(outputParam!=nullptr)
{
std::string checkReturn = outputParam->CheckFileName(true);
if (!checkReturn.empty())
{
otbAppLogWARNING("Check filename: "<<checkReturn);
}
if (useRAM)
{
outputParam->SetRAMValue(ram);
}
outputParam->InitializeWriters();
std::ostringstream progressId;
progressId << "Writing " << outputParam->GetFileName() << "...";
AddProcess(outputParam->GetWriter(), progressId.str());
outputParam->Write();
}
otbAppLogWARNING("Check filename: "<<checkReturn);
}
else if (GetParameterType(key) == ParameterType_OutputVectorData
&& IsParameterEnabled(key) && HasValue(key) )
if (useRAM)
{
Parameter* param = GetParameterByKey(key);
OutputVectorDataParameter* outputParam = dynamic_cast<OutputVectorDataParameter*>(param);
if(outputParam!=nullptr)
{
outputParam->InitializeWriters();
std::ostringstream progressId;
progressId << "Writing " << outputParam->GetFileName() << "...";
AddProcess(outputParam->GetWriter(), progressId.str());
outputParam->Write();
}
outputParam->SetRAMValue(ram);
}
outputParam->InitializeWriters();
std::ostringstream progressId;
progressId << "Writing " << outputParam->GetFileName() << "...";
AddProcess(outputParam->GetWriter(), progressId.str());
outputParam->Write();
}
}
else if (GetParameterType(key) == ParameterType_OutputVectorData
&& IsParameterEnabled(key) && HasValue(key) )
{
Parameter* param = GetParameterByKey(key);
OutputVectorDataParameter* outputParam = dynamic_cast<OutputVectorDataParameter*>(param);
if(outputParam!=nullptr)
{
outputParam->InitializeWriters();
std::ostringstream progressId;
progressId << "Writing " << outputParam->GetFileName() << "...";
AddProcess(outputParam->GetWriter(), progressId.str());
outputParam->Write();
}
}
}
}
int Application::ExecuteAndWriteOutput()
{
m_Chrono.Restart();
// reset the flag m_ExecuteDone
m_ExecuteDone = false;
m_Logger->LogSetupInformation();
int status = this->Execute();
if (status == 0)
{
this->WriteOutput();
}
this->AfterExecuteAndWriteOutputs();
......@@ -1630,5 +1752,91 @@ otbGetParameterImageMacro(ComplexFloatVectorImage);
otbGetParameterImageMacro(ComplexDoubleVectorImage);
bool
Application::ConnectImage(std::string in, Application* app, std::string out)
{
if(app == nullptr)
{
// throw error ?
return false;
}
Parameter* param = GetParameterByKey(in);
InputImageParameter* inParam = dynamic_cast<InputImageParameter*>(param);
if(inParam == nullptr)
{
InputImageListParameter* inListParam = dynamic_cast<InputImageListParameter*>(param);
if (inListParam == nullptr)
{
return false;
}
inListParam->InsertElement(InputImageParameter::New(),inListParam->Size());
inParam = (inListParam->GetNthElement(inListParam->Size() - 1)).GetPointer();
}
param = app->GetParameterByKey(out);
OutputImageParameter* outParam = dynamic_cast<OutputImageParameter*>(param);
if(outParam == nullptr)
{
return false;
}
InputImageParameter::Connector c;
c.app = app;
c.key = out;
c.isMem = true;
inParam->SetConnection(c);
return true;
}
void
Application::PropagateConnectMode(bool isMem)
{
// reset ExecuteDone flag
m_ExecuteDone = false;
std::vector<std::string> paramList = GetParametersKeys(true);
std::unordered_set<Application*> targetApps;
for (std::vector<std::string>::const_iterator it = paramList.begin(); it != paramList.end(); ++it)
{
std::string key = *it;
Parameter* param = GetParameterByKey(key);
InputImageParameter* imgParam = dynamic_cast<InputImageParameter*>(param);
if(imgParam)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgParam->GetConnection().app);
if(targetApp.IsNotNull())
{
imgParam->SetConnectionMode(isMem);
targetApps.insert(targetApp);
}
}
else
{
InputImageListParameter* imgListParam = dynamic_cast<InputImageListParameter*>(param);
if (imgListParam)
{
for (unsigned int i=0 ; i<imgListParam->Size(); i++)
{
Application::Pointer targetApp = otb::DynamicCast<Application>(imgListParam->GetNthElement(i)->GetConnection().app);
if(targetApp.IsNotNull())
{
imgListParam->GetNthElement(i)->SetConnectionMode(isMem);
targetApps.insert(targetApp);
}
}
}
}
}
for (auto &app : targetApps)
{
app->PropagateConnectMode(isMem);
}
}
bool
Application::IsExecuteDone()
{
return m_ExecuteDone;
}
}
}
......@@ -47,7 +47,7 @@ InputImageListParameter
{
assert( image!=nullptr );
InputImageParameter::Pointer p;
InputImageParameter::Pointer p( InputImageParameter::New() );
return FromImage( p, image );
}
......
......@@ -244,7 +244,10 @@ public:
void Init();
void UpdateParameters();
int Execute();
void WriteOutput();
int ExecuteAndWriteOutput();
bool ConnectImage(std::string in, Application* app, std::string out);
void PropagateConnectMode(bool isMem);
void LoadParametersFromXML(const std::string& filename);
void SaveParametersToXML(const std::string& filename);
......
......@@ -22,27 +22,39 @@
# -*- coding: utf-8 -*-
#
# Example on the use of the Rescale
# Example on the use of application connections
#
def test(otb, argv):
app1 = otb.Registry.CreateApplication("Smoothing")
app2 = otb.Registry.CreateApplication("Smoothing")
app3 = otb.Registry.CreateApplication("Smoothing")
app4 = otb.Registry.CreateApplication("ConcatenateImages")
#---------------------------------------------------------------------------
# First run with in-memory connections by default
app1 = otb.Registry.CreateApplication("Smoothing")
app2 = otb.Registry.CreateApplication("Smoothing")
app3 = otb.Registry.CreateApplication("Smoothing")
app4 = otb.Registry.CreateApplication("ConcatenateImages")
app1.IN = argv[1]
app1.Execute()
app1.IN = argv[1]
app1.TYPE = "mean"
app2.SetParameterInputImage("in",app1.GetParameterOutputImage("out"))
app2.Execute()
app2.ConnectImage("in",app1, "out")
app2.TYPE = "anidif"
app3.IN = argv[1]