Skip to content
Snippets Groups Projects
Commit e067ebe0 authored by Otmane Lahlou's avatar Otmane Lahlou
Browse files

ENH : Add static methods IsCurlAvailable and IsCurlMultiAvailable to get the curl status 

      Those methods avoid the use of the C preprocessor directive #ifdef OTB_USE_CURL
      to get the curl status.
parent c678c234
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Code/IO/otbImageIOFactory.cxx
+ 6
− 6
View file @ e067ebe0
......@@ -17,6 +17,7 @@
=========================================================================*/
#include "otbConfigure.h"
#include "otbCurlHelperInterface.h"
#include "otbImageIOFactory.h"
#include "itkMutexLock.h"
#include "itkMutexLockHolder.h"
......@@ -33,9 +34,7 @@
#include "otbJPEG2000ImageIOFactory.h"
#endif
#ifdef OTB_USE_CURL
#include "otbTileMapImageIOFactory.h"
#endif
namespace otb
{
......@@ -75,10 +74,11 @@ ImageIOFactory::RegisterBuiltInFactories()
itk::ObjectFactoryBase::RegisterFactory(JPEG2000ImageIOFactory::New());
#endif
#ifdef OTB_USE_CURL
// TileMap : New format for OTB
itk::ObjectFactoryBase::RegisterFactory(TileMapImageIOFactory::New());
#endif
if (CurlHelperInterface::IsCurlAvailable())
{
// TileMap : New format for OTB
itk::ObjectFactoryBase::RegisterFactory(TileMapImageIOFactory::New());
}
// GDAL : New format for OTB
itk::ObjectFactoryBase::RegisterFactory(GDALImageIOFactory::New());
......
Code/UtilitiesAdapters/CurlAdapters/otbCurlHelper.cxx
+ 235
− 224
View file @ e067ebe0
......@@ -97,7 +97,7 @@ protected:
itkExceptionMacro(<<" otbCurlHelper::CurlResource Curl handle init error.");
}
}
virtual ~CurlResource()
{
curl_easy_cleanup(m_Curl);
......@@ -146,7 +146,7 @@ protected:
{
curl_multi_cleanup(m_Curl);
}
private:
CURLM * m_Curl;
// prevent copying and assignment; not implemented
......@@ -204,81 +204,86 @@ private:
bool CurlHelper::TestUrlAvailability(const std::string& url) const
{
#ifdef OTB_USE_CURL
// Set up a curl resource
CurlResource::Pointer curlResource = CurlResource::New();
// Boolean to return
bool res = false;
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
// Set the dummy write function
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataDummy));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
if (CurlHelperInterface::IsCurlAvailable())
{
// Set up a curl resource
CurlResource::Pointer curlResource = CurlResource::New();
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
// Set the dummy write function
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataDummy));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
// Perform requet
CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
// Perform requet
CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
otbMsgDevMacro(<< "CurlHelper::TestUrlAvailability : curl_easy_perform returned "
<< easyPerformResult << " --> "
<< curl_easy_strerror(easyPerformResult));
// Check the curl_easy_perform return code : actually we tests only
// the availability of the url, so if the file requested size is less
// than 1 byte, and the url is valid it returns CURLE_OK, or if the
// url is valid and the file size is greater than 1 byte it returns
// the error CURLE_FILESIZE_EXCEEDED
if ( easyPerformResult == CURLE_OK || easyPerformResult == CURLE_FILESIZE_EXCEEDED )
otbMsgDevMacro(<< "CurlHelper::TestUrlAvailability : curl_easy_perform returned "
<< easyPerformResult << " --> "
<< curl_easy_strerror(easyPerformResult));
// Check the curl_easy_perform return code : actually we tests only
// the availability of the url, so if the file requested size is less
// than 1 byte, and the url is valid it returns CURLE_OK, or if the
// url is valid and the file size is greater than 1 byte it returns
// the error CURLE_FILESIZE_EXCEEDED
if ( easyPerformResult == CURLE_OK || easyPerformResult == CURLE_FILESIZE_EXCEEDED )
{
return true;
}
return false;
}
else
{
return true;
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
}
return res;
#else
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
return false;
#endif
}
int CurlHelper::RetrieveUrlInMemory(const std::string& url, std::string& output) const
{
#ifdef OTB_USE_CURL
otbMsgDevMacro(<< "Retrieving: " << url);
CURLcode res = CURLE_OK;
CurlResource::Pointer curlResource = CurlResource::New();
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.c_str()));
// Set 5s timeout
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, 10));
// Use our writing static function to avoid file descriptor
// pointer crash on windows
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToStringStream));
// Say the file where to write the received data
std::ostringstream* outputStream = new std::ostringstream;
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA, (void*) outputStream));
if(CurlHelperInterface::IsCurlAvailable())
{
otbMsgDevMacro(<< "Retrieving: " << url);
CURLcode res = CURLE_OK;
CurlResource::Pointer curlResource = CurlResource::New();
// Perform request
otbCurlCall(curl_easy_perform(curlResource->GetCurlResource()));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.c_str()));
// Save output
output = outputStream->str();
otbMsgDevMacro("curl output : " << output);
// Set 5s timeout
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, 10));
// Clean up
delete outputStream;
// Use our writing static function to avoid file descriptor
// pointer crash on windows
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToStringStream));
otbMsgDevMacro(<< " -> " << res);
return res;
#else
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
// Say the file where to write the received data
std::ostringstream* outputStream = new std::ostringstream;
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA, (void*) outputStream));
// Perform request
otbCurlCall(curl_easy_perform(curlResource->GetCurlResource()));
// Save output
output = outputStream->str();
otbMsgDevMacro("curl output : " << output);
// Clean up
delete outputStream;
otbMsgDevMacro(<< " -> " << res);
return res;
}
else
{
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
}
return -1;
#endif
}
int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string filename) const
......@@ -288,192 +293,199 @@ int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string fi
int CurlHelper::RetrieveFile(const std::string& urlString, std::string filename) const
{
#ifdef OTB_USE_CURL
otbMsgDevMacro(<< "Retrieving: " << urlString);
CURLcode res = CURLE_OK;
if(CurlHelperInterface::IsCurlAvailable())
{
otbMsgDevMacro(<< "Retrieving: " << urlString);
CURLcode res = CURLE_OK;
CurlResource::Pointer curlResource = CurlResource::New();
CurlResource::Pointer curlResource = CurlResource::New();
CurlFileDescriptorResource::Pointer output_file = CurlFileDescriptorResource::New();
output_file->OpenFile(filename.c_str());
CurlFileDescriptorResource::Pointer output_file = CurlFileDescriptorResource::New();
output_file->OpenFile(filename.c_str());
char url[256];
strcpy(url, urlString.c_str());
char url[256];
strcpy(url, urlString.c_str());
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url));
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url));
// Set 10s timeout
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, 10));
// Set 10s timeout
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, 10));
// Use our writing static function to avoid file descriptor
// pointer crash on windows
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToFile));
// Say the file where to write the received data
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA,
(void*) output_file->GetFileResource()));
// Use our writing static function to avoid file descriptor
// pointer crash on windows
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToFile));
// Say the file where to write the received data
otbCurlCall(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA,
(void*) output_file->GetFileResource()));
otbCurlCall(curl_easy_perform(curlResource->GetCurlResource()));
otbCurlCall(curl_easy_perform(curlResource->GetCurlResource()));
otbMsgDevMacro(<< " -> " << res);
return res;
#else
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
otbMsgDevMacro(<< " -> " << res);
return res;
}
else
{
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
}
return -1;
#endif
}
int CurlHelper::RetrieveFileMulti(const std::vector<std::string>& listURLs,
const std::vector<std::string>& listFilename,
int maxConnect) const
{
#ifdef OTB_USE_CURL
#ifdef OTB_CURL_MULTI_AVAILABLE
otbMsgDevMacro(<< "Using curl multi");
// Initialize curl handle resource
CurlMultiResource::Pointer multiHandle = CurlMultiResource::New();
std::vector<CurlResource::Pointer> listCurlHandles;
std::vector<CurlFileDescriptorResource::Pointer> listFiles;
std::vector<std::string>::const_iterator filename;
filename = listFilename.begin();
while (filename != listFilename.end())
if(CurlHelperInterface::IsCurlAvailable())
{
CurlFileDescriptorResource::Pointer lOutputFile = CurlFileDescriptorResource::New();
lOutputFile->OpenFile((*filename).c_str());
// Add file to vector
listFiles.push_back(lOutputFile);
++filename;
}
if(CurlHelperInterface::IsCurlMultiAvailable())
{
otbMsgDevMacro(<< "Using curl multi");
listCurlHandles.clear();
// Initialize curl handle resource
CurlMultiResource::Pointer multiHandle = CurlMultiResource::New();
std::vector<std::string>::const_iterator url;
std::vector<CurlFileDescriptorResource::Pointer>::const_iterator file;
url = listURLs.begin();
file = listFiles.begin();
while ((url != listURLs.end()) && (file != listFiles.end()))
{
otbMsgDevMacro(<< "Retrieving: " << (*url).data());
CurlResource::Pointer lEasyHandle = CurlResource::New();
// Param easy handle
// the otbCurlCall is not used here cause we are not using the
// CurlResource class : Using the CurlResource class in the loop
// will make the object CurlResource destroyed at the end of each
// loop, making the multi handle not working.
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_URL, (*url).data()));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToFile));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEDATA,
std::vector<CurlResource::Pointer> listCurlHandles;
std::vector<CurlFileDescriptorResource::Pointer> listFiles;
std::vector<std::string>::const_iterator filename;
filename = listFilename.begin();
while (filename != listFilename.end())
{
CurlFileDescriptorResource::Pointer lOutputFile = CurlFileDescriptorResource::New();
lOutputFile->OpenFile((*filename).c_str());
// Add file to vector
listFiles.push_back(lOutputFile);
++filename;
}
listCurlHandles.clear();
std::vector<std::string>::const_iterator url;
std::vector<CurlFileDescriptorResource::Pointer>::const_iterator file;
url = listURLs.begin();
file = listFiles.begin();
while ((url != listURLs.end()) && (file != listFiles.end()))
{
otbMsgDevMacro(<< "Retrieving: " << (*url).data());
CurlResource::Pointer lEasyHandle = CurlResource::New();
// Param easy handle
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_URL, (*url).data()));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEFUNCTION,
&Self::CallbackWriteDataToFile));
otbCurlCall(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEDATA,
(void*) (*file)->GetFileResource()));
// Add easy handle to multi handle
otbCurlMultiCall(curl_multi_add_handle(multiHandle->GetCurlMultiResource(), lEasyHandle->GetCurlResource()));
// Add hanle to vector
listCurlHandles.push_back(lEasyHandle);
++url;
++file;
}
//fetch tiles
// Configure multi handle - set the maximum connections
otbCurlMultiCall(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_MAXCONNECTS, maxConnect));
otbCurlMultiCall(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_PIPELINING, 0));
// Perform
int lStillRunning;
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
// Now get that URL
while (lStillRunning)
{
struct timeval timeout;
int rc; // Return code
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to play around with */
timeout.tv_sec = 0;
timeout.tv_usec = 1;
/* get file descriptors from the transfers */
otbCurlMultiCall(curl_multi_fdset(multiHandle->GetCurlMultiResource(), &fdread, &fdwrite, &fdexcep, &maxfd));
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
switch (rc)
{
case -1:
/* select error */
break;
case 0:
/* timeout */
default:
/* timeout or readable/writable sockets */
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
break;
// Add easy handle to multi handle
otbCurlMultiCall(curl_multi_add_handle(multiHandle->GetCurlMultiResource(), lEasyHandle->GetCurlResource()));
// Add hanle to vector
listCurlHandles.push_back(lEasyHandle);
++url;
++file;
}
//fetch tiles
// Configure multi handle - set the maximum connections
otbCurlMultiCall(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_MAXCONNECTS, maxConnect));
otbCurlMultiCall(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_PIPELINING, 0));
// Perform
int lStillRunning;
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
// Now get that URL
while (lStillRunning)
{
struct timeval timeout;
int rc; // Return code
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to play around with */
timeout.tv_sec = 0;
timeout.tv_usec = 1;
/* get file descriptors from the transfers */
otbCurlMultiCall(curl_multi_fdset(multiHandle->GetCurlMultiResource(), &fdread, &fdwrite, &fdexcep, &maxfd));
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
switch (rc)
{
case -1:
/* select error */
break;
case 0:
/* timeout */
default:
/* timeout or readable/writable sockets */
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
break;
}
}
int remaining_msgs = 1;
int error = 0;
CURLMsg *msg;
while (remaining_msgs)
{
msg = curl_multi_info_read(multiHandle->GetCurlMultiResource(), &remaining_msgs);
if (msg != NULL)
{
if (CURLE_OK != msg->data.result) error = 1;
}
}
if (error != 0)
{
itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
}
// Cleanup
listFiles.clear();
listCurlHandles.clear();
}
}
int remaining_msgs = 1;
int error = 0;
CURLMsg *msg;
while (remaining_msgs)
{
msg = curl_multi_info_read(multiHandle->GetCurlMultiResource(), &remaining_msgs);
if (msg != NULL)
else
{
if (CURLE_OK != msg->data.result) error = 1;
}
}
if (error != 0)
//fallback on non curl multi
otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");
std::vector<std::string>::const_iterator url;
std::vector<std::string>::const_iterator file;
url = listURLs.begin();
file = listFilename.begin();
int res = 0;
int resTmp = -1;
while ((url != listURLs.end()) && (file != listFilename.end()))
{
resTmp = RetrieveFile(*url, *file);
if (res == 0) res = resTmp;
++url;
++file;
}
return res;
} // end else CurlMultiAvailable
} // else CurlAvailabel
else
{
itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
}
// Cleanup
listFiles.clear();
listCurlHandles.clear();
#else
//fallback on non curl multi
otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");
std::vector<std::string>::const_iterator url;
std::vector<std::string>::const_iterator file;
url = listURLs.begin();
file = listFilename.begin();
int res = 0;
int resTmp = -1;
while ((url != listURLs.end()) && (file != listFilename.end()))
{
resTmp = RetrieveFile(*url, *file);
if (res == 0) res = resTmp;
++url;
++file;
}
return res;
#endif
#else
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
return -1;
#endif
}
size_t CurlHelper::CallbackWriteDataToFile(void* ptr, size_t size, size_t nmemb, void* data)
......@@ -508,7 +520,6 @@ size_t CurlHelper::CallbackWriteDataToStringStream(void *ptr, size_t size, size_
return size * nmemb;
}
size_t CurlHelper::CallbackWriteDataDummy(void *ptr, size_t size, size_t nmemb, void *data)
{
return size * nmemb;
......
Code/UtilitiesAdapters/CurlAdapters/otbCurlHelperInterface.cxx 0 → 100644
+ 46
− 0
View file @ e067ebe0
/*=========================================================================
Program: ORFEO Toolbox
Language: C++
Date: $Date$
Version: $Revision$
Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
See OTBCopyright.txt for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notices for more information.
=========================================================================*/
#include "otbCurlHelperInterface.h"
#include "otbConfigure.h"
namespace otb {
bool
CurlHelperInterface::IsCurlAvailable()
{
bool curlAvailable = false;
#ifdef OTB_USE_CURL
curlAvailable = true;
#endif
return curlAvailable;
}
bool
CurlHelperInterface::IsCurlMultiAvailable()
{
bool curlMultiAvailable = false;
#ifdef OTB_CURL_MULTI_AVAILABLE
curlMultiAvailable = true;
#endif
return curlMultiAvailable;
}
}
Code/UtilitiesAdapters/CurlAdapters/otbCurlHelperInterface.h
+ 5
− 0
View file @ e067ebe0
......@@ -22,6 +22,7 @@
#include "itkObject.h"
#include "itkObjectFactory.h"
namespace otb
{
/**
......@@ -54,6 +55,10 @@ public:
virtual int RetrieveFileMulti(const std::vector<std::string>& listURLs,
const std::vector<std::string>& listFiles,
int maxConnect) const = 0;
static bool IsCurlAvailable();
static bool IsCurlMultiAvailable();
protected:
CurlHelperInterface() {}
virtual ~CurlHelperInterface() {}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment