From 3d4ab1ff163023cc3d85712f95dd1a17c500b439 Mon Sep 17 00:00:00 2001
From: Ludovic Hussonnois <ludovic.hussonnois@c-s.fr>
Date: Wed, 8 Feb 2017 16:06:08 +0100
Subject: [PATCH] BUG: Try fix fft random segfault.
Terminate MPI before the end of the main instead of the destructor.
Also clear FFTW threads at the end of DoExecute() not in the destructor.
---
.../app/otbDomainTransform.cxx | 8 +-
Modules/IO/TestKernel/include/otbTestMain.h | 1 +
Modules/MPI/MPIConfig/include/otbMPIConfig.h | 5 +
Modules/MPI/MPIConfig/src/otbMPIConfig.cxx | 121 ++++++++++--------
.../src/otbApplicationLauncherCommandLine.cxx | 18 +--
5 files changed, 86 insertions(+), 67 deletions(-)
diff --git a/Modules/Applications/AppDomainTransform/app/otbDomainTransform.cxx b/Modules/Applications/AppDomainTransform/app/otbDomainTransform.cxx
index 2f611618e2..41ee75c1b2 100644
--- a/Modules/Applications/AppDomainTransform/app/otbDomainTransform.cxx
+++ b/Modules/Applications/AppDomainTransform/app/otbDomainTransform.cxx
@@ -27,6 +27,7 @@
#include <itkInverseFFTImageFilter.h>
#include <itkUnaryFunctorImageFilter.h>
#include <itkFFTShiftImageFilter.h>
+#include <itkFFTWGlobalConfiguration.h>
#include "otbComplexToVectorImageCastFilter.h"
@@ -79,11 +80,9 @@ private:
~DomainTransform() ITK_OVERRIDE
{
- // Clean up all threads allocated threads.
- CleanupFFTThreads();
}
- void CleanupFFTThreads()
+ void CleanupFFTWThreads()
{
std::cout << "Start cleanup threads" << std::endl;
// This is a trick to make sure fftw will cleanup its threads when application
@@ -374,6 +373,9 @@ private:
SetParameterOutputImage<TOutputImage>("out", invFilter->GetOutput());
}
}
+
+ // at the end, cleanup FFTW Threads
+ CleanupFFTWThreads();
}
template<otb::Wavelet::Wavelet TWaveletOperator>
diff --git a/Modules/IO/TestKernel/include/otbTestMain.h b/Modules/IO/TestKernel/include/otbTestMain.h
index 42f7e894aa..c30763f41a 100644
--- a/Modules/IO/TestKernel/include/otbTestMain.h
+++ b/Modules/IO/TestKernel/include/otbTestMain.h
@@ -424,6 +424,7 @@ int main(int ac, char* av[])
std::cout << "------------- End control baseline tests -------------" << std::endl;
#ifdef OTB_USE_MPI
+ otb::MPIConfig::Instance()->terminate();
}
#endif
return result;
diff --git a/Modules/MPI/MPIConfig/include/otbMPIConfig.h b/Modules/MPI/MPIConfig/include/otbMPIConfig.h
index 3b53299fdd..c5bd1ab1e8 100644
--- a/Modules/MPI/MPIConfig/include/otbMPIConfig.h
+++ b/Modules/MPI/MPIConfig/include/otbMPIConfig.h
@@ -53,6 +53,9 @@ public:
/** Initialize MPI Processus */
void Init(int& argc, char** &argv, bool abortOnException = true);
+ /** Shuts down the MPI environment. */
+ void terminate();
+
/** Abort all MPI processus. */
void abort(int errCode);
@@ -85,6 +88,8 @@ private:
bool m_abortOnException;
// Boolean to test if the MPI environment is initialized
bool m_initialized;
+ // Boolean to test if the MPI environment is terminated
+ bool m_terminated;
static Pointer m_Singleton;
};
diff --git a/Modules/MPI/MPIConfig/src/otbMPIConfig.cxx b/Modules/MPI/MPIConfig/src/otbMPIConfig.cxx
index 5422797b1b..6d53747f4b 100644
--- a/Modules/MPI/MPIConfig/src/otbMPIConfig.cxx
+++ b/Modules/MPI/MPIConfig/src/otbMPIConfig.cxx
@@ -80,74 +80,91 @@ MPIConfig::MPIConfig()
: m_MyRank(0),
m_NbProcs(0),
m_abortOnException(true),
- m_initialized(false)
+ m_initialized(false),
+ m_terminated(false)
{
}
/** Shuts down the MPI environment. */
MPIConfig::~MPIConfig()
{
- if (m_initialized)
- {
- if (std::uncaught_exception() && m_abortOnException)
- {
- abort(EXIT_FAILURE);
- }
- else
- {
- std::cout << "Start Finalize MPI" << std::endl;
- std::flush(std::cout);
- int finalized;
- OTB_MPI_CHECK_RESULT(MPI_Finalized,(&finalized));
- if (!finalized)
- {
- std::cout << "Start 2nd End Finalize MPI" << std::endl;
- OTB_MPI_CHECK_RESULT(MPI_Finalize,());
- std::cout << "End 2nd Finalize MPI" << std::endl;
- }
- std::cout << "End Finalize MPI" << std::endl;
- std::flush(std::cout);
- }
- }
+ terminate();
}
/** Initialize MPI environment */
-void MPIConfig::Init(int& argc, char** &argv, bool abortOnException) {
- // Abort on exception
- m_abortOnException = abortOnException;
- // Initialize
- int initialized;
- OTB_MPI_CHECK_RESULT(MPI_Initialized,(&initialized));
- m_initialized = (initialized == 1);
- if (!m_initialized) {
- OTB_MPI_CHECK_RESULT(MPI_Init, (&argc, &argv));
- m_initialized = true;
- }
- // Get MPI rank
- int irank = 0;
- OTB_MPI_CHECK_RESULT(MPI_Comm_rank, (MPI_COMM_WORLD , &irank));
+void MPIConfig::Init(int &argc, char **&argv, bool abortOnException) {
+ if( !m_terminated )
+ {
+ std::cout << "MPI Initialization : " << this << std::endl;
+ std::flush(std::cout);
+ // Abort on exception
+ m_abortOnException = abortOnException;
+ // Initialize
+ int initialized;
+ OTB_MPI_CHECK_RESULT( MPI_Initialized, ( &initialized ));
+ m_initialized = ( initialized == 1 );
+ if( !m_initialized )
+ {
+ OTB_MPI_CHECK_RESULT( MPI_Init, ( &argc, &argv ));
+ m_initialized = true;
+ }
+ // Get MPI rank
+ int irank = 0;
+ OTB_MPI_CHECK_RESULT( MPI_Comm_rank, ( MPI_COMM_WORLD, &irank ));
- if(irank<0)
- {
- logError("Negative MPI rank");
- abort(EXIT_FAILURE);
- }
+ if( irank < 0 )
+ {
+ logError( "Negative MPI rank" );
+ abort( EXIT_FAILURE );
+ }
- m_MyRank = static_cast<unsigned int>(irank);
+ m_MyRank = static_cast<unsigned int>(irank);
- // Get MPI NbProocs
+ // Get MPI NbProocs
- int inbprocs=0;
+ int inbprocs = 0;
- OTB_MPI_CHECK_RESULT(MPI_Comm_size, (MPI_COMM_WORLD , &inbprocs));
+ OTB_MPI_CHECK_RESULT( MPI_Comm_size, ( MPI_COMM_WORLD, &inbprocs ));
- if(inbprocs<1)
- {
- logError("Negative or null number of processes");
- abort(EXIT_FAILURE);
- }
+ if( inbprocs < 1 )
+ {
+ logError( "Negative or null number of processes" );
+ abort( EXIT_FAILURE );
+ }
- m_NbProcs = static_cast<unsigned int>(inbprocs);
+ m_NbProcs = static_cast<unsigned int>(inbprocs);
+ }
+}
+
+/** Shuts down the MPI environment. */
+void MPIConfig::terminate()
+{
+ if( m_initialized && !m_terminated )
+ {
+ std::cout << "Terminate : " << this << std::endl;
+ std::flush(std::cout);
+ if( std::uncaught_exception() && m_abortOnException )
+ {
+ std::cout << "Abord" << std::endl;
+ abort( EXIT_FAILURE );
+ }
+ else
+ {
+ std::cout << "Start Finalize MPI" << std::endl;
+ std::flush( std::cout );
+ int finalized;
+ OTB_MPI_CHECK_RESULT( MPI_Finalized, ( &finalized ));
+ if( !finalized )
+ {
+ std::cout << "Start 2nd End Finalize MPI" << std::endl;
+ //OTB_MPI_CHECK_RESULT( MPI_Finalize, ( ));
+ std::cout << "End 2nd Finalize MPI" << std::endl;
+ }
+ std::cout << "End Finalize MPI" << std::endl;
+ std::flush( std::cout );
+ }
+ m_terminated = true;
+ }
}
void MPIConfig::abort(int errCode)
diff --git a/Modules/Wrappers/CommandLine/src/otbApplicationLauncherCommandLine.cxx b/Modules/Wrappers/CommandLine/src/otbApplicationLauncherCommandLine.cxx
index 05d439d7fa..3d66679172 100644
--- a/Modules/Wrappers/CommandLine/src/otbApplicationLauncherCommandLine.cxx
+++ b/Modules/Wrappers/CommandLine/src/otbApplicationLauncherCommandLine.cxx
@@ -303,19 +303,13 @@ int main(int argc, char* argv[])
LauncherType::Pointer launcher = LauncherType::New();
//if (launcher->Load(exp) == true)
- if (launcher->Load(vexp) == true)
- {
- if (launcher->ExecuteAndWriteOutput() == false)
- {
- return EXIT_FAILURE;
- }
- }
- else
- {
- return EXIT_FAILURE;
- }
+ bool sucess = launcher->Load(vexp) && launcher->ExecuteAndWriteOutput();
- return EXIT_SUCCESS;
+ // shutdown MPI after application finished
+ #ifdef OTB_USE_MPI
+ otb::MPIConfig::Instance()->terminate();
+ #endif
+ return sucess ? EXIT_SUCCESS : EXIT_FAILURE;
}
const std::string GetChildNodeTextOf(TiXmlElement *parentElement, std::string key)
--
GitLab