diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5b8b35a05faf7433b7ce7a90edf195216d60a317..8bcb20684652b58bc37fa046fbdaf7ef8a6dc3d6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -366,6 +366,10 @@ IF(OTB_USE_CURL)
INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
+ # TODO add test to check if curl multi is available
+ OPTION(OTB_CURL_MULTI_AVAILABLE "Whether or not the curl has curlmulti capabilities" ON)
+ MARK_AS_ADVANCED(OTB_CURL_MULTI_AVAILABLE)
+ ADD_DEFINITIONS(-DOTB_CURL_MULTI_AVAILABLE)
ENDIF(OTB_USE_CURL)
#-------------------------------
diff --git a/Code/IO/otbCurlHelper.cxx b/Code/IO/otbCurlHelper.cxx
index 720aa2aab2ba478738e0d438a639129d8db91652..660a3bb35dab2781607fe062a17d77bbc3bc8c59 100644
--- a/Code/IO/otbCurlHelper.cxx
+++ b/Code/IO/otbCurlHelper.cxx
@@ -17,7 +17,7 @@
=========================================================================*/
#include "otbCurlHelper.h"
-
+#include "otbMacro.h"
#ifdef OTB_USE_CURL
#include <curl/curl.h>
@@ -58,6 +58,11 @@ int CurlHelper::TestUrlAvailability(const std::string& url) const
}
int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string filename) const
+{
+ return RetrieveFile(urlStream.str(), filename);
+}
+
+int CurlHelper::RetrieveFile(const std::string& urlString, std::string filename) const
{
#ifdef OTB_USE_CURL
CURL * curl;
@@ -67,7 +72,7 @@ int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string fi
curl = curl_easy_init();
char url[256];
- strcpy(url, urlStream.str().data());
+ strcpy(url, urlString.data());
// std::cout << url << std::endl;
if (curl)
@@ -92,5 +97,183 @@ int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string fi
#endif
}
+
+int CurlHelper::RetrieveFileMulti(const std::vector<std::string>& listURLs,
+ const std::vector<std::string>& listFilename,
+ int maxConnect) const
+{
+#ifdef OTB_USE_CURL
+#ifdef OTB_CURL_MULTI_AVAILABLE
+ otbMsgDevMacro(<< "Using curl multi");
+ std::string m_Browser = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11";
+
+ CURLM * multiHandle;
+ std::vector<CURL *> listCurlHandles;
+ std::vector<FILE *> listFiles;
+
+ std::vector<std::string>::const_iterator filename;
+ filename = listFilename.begin();
+ while (filename != listFilename.end() )
+ {
+ FILE* lOutputFile = fopen((*filename).c_str(), "w");
+ if (lOutputFile == NULL)
+ {
+ itkExceptionMacro(<< "otbCurlHelper: bad file name: " << (*filename).c_str());
+ }
+
+ // Add file to vector
+ listFiles.push_back(lOutputFile);
+ ++filename;
+ }
+
+ // Initialize curl multi handle
+ multiHandle = curl_multi_init();
+
+ if (!multiHandle)
+ {
+ itkExceptionMacro(<< "otbCurlHelper: Curl multi handle init error.");
+ }
+
+ listCurlHandles.clear();
+
+ // Initialize curl handle
+
+ std::vector<std::string>::const_iterator url;
+ std::vector<FILE *>::const_iterator file;
+ url = listURLs.begin();
+ file = listFiles.begin();
+ while ( (url != listURLs.end()) && (file != listFiles.end() ))
+ {
+ CURL * lEasyHandle;
+ lEasyHandle = curl_easy_init();
+
+ if (!lEasyHandle)
+ {
+ itkExceptionMacro(<< "otbCurlHelper: Curl easy handle init error.");
+ }
+
+ // Param easy handle
+ curl_easy_setopt(lEasyHandle, CURLOPT_USERAGENT, m_Browser.data());
+ curl_easy_setopt(lEasyHandle, CURLOPT_URL, (*url).data());
+ curl_easy_setopt(lEasyHandle, CURLOPT_WRITEDATA, *file);
+
+ // Add easy handle to multi handle
+ curl_multi_add_handle(multiHandle, lEasyHandle);
+
+ // Add hanle to vector
+ listCurlHandles.push_back(lEasyHandle);
+ ++url;
+ ++file;
+ }
+
+//fetch tiles
+ // Configure multi handle - set the maximum connections
+ curl_multi_setopt(multiHandle, CURLMOPT_MAXCONNECTS, maxConnect);
+ curl_multi_setopt(multiHandle, CURLMOPT_PIPELINING, 0);
+
+ // Perform
+ int lStillRunning;
+
+ while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle, &lStillRunning));
+
+ // Now get that URL
+ while (lStillRunning)
+ {
+ struct timeval timeout;
+ int rc; // Return code
+
+ fd_set fdread;
+ fd_set fdwrite;
+ fd_set fdexcep;
+ int maxfd;
+
+ FD_ZERO(&fdread);
+ FD_ZERO(&fdwrite);
+ FD_ZERO(&fdexcep);
+
+ /* set a suitable timeout to play around with */
+ timeout.tv_sec = 0;
+ timeout.tv_usec = 1;
+
+ /* get file descriptors from the transfers */
+ curl_multi_fdset(multiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+ rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+ switch (rc)
+ {
+ case -1:
+ /* select error */
+ break;
+ case 0:
+ /* timeout */
+ default:
+ /* timeout or readable/writable sockets */
+ while (
+ CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle, &lStillRunning)
+ )
+ ;
+ break;
+ }
+ }
+
+ int remaining_msgs = 1;
+ int error = 0;
+ CURLMsg *msg;
+ while (remaining_msgs)
+ {
+ msg = curl_multi_info_read(multiHandle, &remaining_msgs);
+ if (msg != NULL)
+ {
+ if (CURLE_OK != msg->data.result) error = 1;
+ }
+ }
+
+ if (error != 0)
+ {
+ itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
+ }
+
+
+// Cleanup
+
+ // Close files
+ for (unsigned int currentFile = 0; currentFile < listFiles.size(); currentFile++)
+ {
+ fclose(listFiles[currentFile]);
+ }
+ listFiles.clear();
+
+ // Cleanup easy handles
+ for (unsigned int currentHandle = 0; currentHandle < listCurlHandles.size(); currentHandle++)
+ {
+ curl_easy_cleanup(listCurlHandles[currentHandle]);
+ }
+ listCurlHandles.clear();
+
+ // Cleanup multi handle
+ curl_multi_cleanup(multiHandle);
+
+
+
+#else
+ //fallback on non curl multi
+ otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");
+ std::vector<std::string>::const_iterator url;
+ std::vector<FILE *>::const_iterator file;
+ url = listURLs.begin();
+ file = listFiles.begin();
+ while ( (url != listURLs.end()) && (file != listFiles.end() ))
+ {
+ RetrieveFile(*url, *file);
+ }
+#endif
+#else
+ otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
+ return -1;
+#endif
+}
+
+
}
diff --git a/Code/IO/otbCurlHelper.h b/Code/IO/otbCurlHelper.h
index b3152e6e2d5cbf6f0aa6ba9d8185f716184432ce..d535c496c5e435f6439b82b5773722ae07f2fcda 100644
--- a/Code/IO/otbCurlHelper.h
+++ b/Code/IO/otbCurlHelper.h
@@ -43,7 +43,11 @@ public:
int TestUrlAvailability(const std::string& url) const;
int RetrieveFile(const std::ostringstream& urlStream, std::string filename) const;
+ int RetrieveFile(const std::string& urlString, std::string filename) const;
+ int RetrieveFileMulti(const std::vector<std::string>& listURLs,
+ const std::vector<std::string>& listFiles,
+ int maxConnect) const;
protected:
CurlHelper() {}
~CurlHelper() {}
diff --git a/Code/IO/otbTileMapImageIO.cxx b/Code/IO/otbTileMapImageIO.cxx
index db38e270a4fc182986ac129542c5e3881bcf514a..0232a7046997b2fc79d1caf1c6d7a51700338b02 100644
--- a/Code/IO/otbTileMapImageIO.cxx
+++ b/Code/IO/otbTileMapImageIO.cxx
@@ -38,6 +38,8 @@
#include "base/ossimFilename.h"
#include "itkTimeProbe.h"
+#include "otbCurlHelper.h"
+
namespace otb
{
@@ -78,8 +80,6 @@ TileMapImageIO::TileMapImageIO()
m_FileNameIsServerName = false;
- // Initialize browser
- m_Browser = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11";
// Set maximum of connections to 10
m_MaxConnect = 10;
@@ -152,17 +152,9 @@ void TileMapImageIO::Read(void* buffer)
int nTilesX = (int) ceil(totSamples / 256.) + 1;
int nTilesY = (int) ceil(totLines / 256.) + 1;
- // Initialize curl multi handle
- m_MultiHandle = curl_multi_init();
-
- if (!m_MultiHandle)
- {
- itkExceptionMacro(<< "Tile Map IO : Curl mutli handle init error.");
- }
// Clear vectors
- m_ListCurlHandles.clear();
- m_ListFiles.clear();
+ m_ListFilename.clear();
m_ListURLs.clear();
m_ListTiles.clear();
@@ -182,17 +174,16 @@ void TileMapImageIO::Read(void* buffer)
// Try to read tile from cache
if (!this->CanReadFromCache(m_ListTiles.back().filename))
{
- // Generate curl handle for this tile
- this->GenerateCURLHandle(m_ListTiles.back());
+ this->GenerateURL(m_ListTiles.back().x, m_ListTiles.back().y);
+ m_ListFilename.push_back(m_ListTiles.back().filename);
}
}
}
- // Fetch tiles from net
- this->FetchTiles();
+ CurlHelper::Pointer curlHelper = CurlHelper::New();
+ curlHelper->RetrieveFileMulti(m_ListURLs, m_ListFilename, m_MaxConnect);
- // Cleanup datas use to download tiles
- this->Cleanup();
+ m_ListURLs.clear();
// Generate buffer
this->GenerateBuffer(p);
@@ -248,48 +239,6 @@ bool TileMapImageIO::CanReadFromCache(std::string filename)
return lCanRead;
}
-/*
- * This method generate curl handles and add to multi handle
- */
-void TileMapImageIO::GenerateCURLHandle(TileNameAndCoordType tileInfo)
-{
- // Generate URL
- this->GenerateURL(tileInfo.x, tileInfo.y);
-
-
- // Create file
- FILE* lOutputFile = fopen(tileInfo.filename.c_str(), "w");
-
- if (lOutputFile == NULL)
- {
- itkExceptionMacro(<< "TileMap read : bad file name.");
- }
-
- // Add file to vector
- m_ListFiles.push_back(lOutputFile);
-
- // Initialize curl handle
- CURL * lEasyHandle;
- lEasyHandle = curl_easy_init();
-
- if (!lEasyHandle)
- {
- itkExceptionMacro(<< "Tile Map IO : Curl easy handle init error.");
- }
-
-
- // Param easy handle
- curl_easy_setopt(lEasyHandle, CURLOPT_USERAGENT, m_Browser.data());
- curl_easy_setopt(lEasyHandle, CURLOPT_URL, m_ListURLs.back().data());
- curl_easy_setopt(lEasyHandle, CURLOPT_WRITEDATA, m_ListFiles.back());
-
- // Add easy handle to multi handle
- curl_multi_add_handle(m_MultiHandle, lEasyHandle);
-
- // Add hanle to vector
- m_ListCurlHandles.push_back(lEasyHandle);
-}
-
/*
* This method generate URLs
*/
@@ -347,108 +296,6 @@ void TileMapImageIO::GenerateURL(double x, double y)
m_ListURLs.push_back(urlStream.str());
}
-/*
- * This method perform curl multi handle
- */
-void TileMapImageIO::FetchTiles()
-{
- // Configure multi handle - set the maximum connections
- curl_multi_setopt(m_MultiHandle, CURLMOPT_MAXCONNECTS, m_MaxConnect);
- curl_multi_setopt(m_MultiHandle, CURLMOPT_PIPELINING, 0);
-
- // Perform
- int lStillRunning;
-
- while (
- CURLM_CALL_MULTI_PERFORM == curl_multi_perform(m_MultiHandle, &lStillRunning)
- )
- ;
-
- // Now get that URL
- while (lStillRunning)
- {
- struct timeval timeout;
- int rc; // Return code
-
- fd_set fdread;
- fd_set fdwrite;
- fd_set fdexcep;
- int maxfd;
-
- FD_ZERO(&fdread);
- FD_ZERO(&fdwrite);
- FD_ZERO(&fdexcep);
-
- /* set a suitable timeout to play around with */
- timeout.tv_sec = 0;
- timeout.tv_usec = 1;
-
- /* get file descriptors from the transfers */
- curl_multi_fdset(m_MultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
-
- rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
-
- switch (rc)
- {
- case -1:
- /* select error */
- break;
- case 0:
- /* timeout */
- default:
- /* timeout or readable/writable sockets */
- while (
- CURLM_CALL_MULTI_PERFORM == curl_multi_perform(m_MultiHandle, &lStillRunning)
- )
- ;
- break;
- }
- }
-
- int remaining_msgs = 1;
- int error = 0;
- CURLMsg *msg;
- while (remaining_msgs)
- {
- msg = curl_multi_info_read(m_MultiHandle, &remaining_msgs);
- if (msg != NULL)
- {
- if (CURLE_OK != msg->data.result) error = 1;
- }
- }
-
- if (error != 0)
- {
- itkExceptionMacro(<< "TileMapImageIO : Error occurs while perform Multi handle");
- }
-}
-
-/*
- * This method cleanup datas
- */
-void TileMapImageIO::Cleanup()
-{
- // Close files
- for (unsigned int currentFile = 0; currentFile < m_ListFiles.size(); currentFile++)
- {
- fclose(m_ListFiles[currentFile]);
- }
- m_ListFiles.clear();
-
- // Cleanup easy handles
- for (unsigned int currentHandle = 0; currentHandle < m_ListCurlHandles.size(); currentHandle++)
- {
- curl_easy_cleanup(m_ListCurlHandles[currentHandle]);
- }
- m_ListCurlHandles.clear();
-
- // Cleanup multi handle
- curl_multi_cleanup(m_MultiHandle);
-
- // Cleanup url vector
- m_ListURLs.clear();
-}
-
/*
* This method generate the output buffer
*/
diff --git a/Code/IO/otbTileMapImageIO.h b/Code/IO/otbTileMapImageIO.h
index d8734e23053833959e3ad711aabe45baee4687bf..5dca3ed87228deb7c64a93263397d0cc77dddaa1 100644
--- a/Code/IO/otbTileMapImageIO.h
+++ b/Code/IO/otbTileMapImageIO.h
@@ -30,9 +30,6 @@
#include "itkImageIOBase.h"
#include "otbImageRegionTileMapSplitter.h"
-/* Curl Library*/
-#include <curl/curl.h>
-
namespace otb
{
@@ -190,19 +187,13 @@ private:
/** CURL Multi */
void GenerateTileInfo(double x, double y, int numTileX, int numTileY);
bool CanReadFromCache(std::string filename);
- void GenerateCURLHandle(TileNameAndCoordType tileInfo);
void GenerateURL(double x, double y);
- void FetchTiles();
- void Cleanup();
void GenerateBuffer(unsigned char * p);
void ReadTile(std::string filename, void * buffer);
- CURLM * m_MultiHandle;
- std::vector<CURL *> m_ListCurlHandles;
- std::vector<FILE *> m_ListFiles;
+ std::vector<std::string> m_ListFilename;
std::vector<std::string> m_ListURLs;
std::vector<TileNameAndCoordType> m_ListTiles;
- std::string m_Browser;
int m_MaxConnect;
/** Byte per pixel pixel */
diff --git a/OTBConfig.cmake.in b/OTBConfig.cmake.in
index c37c000939598bf616f91d65070af61e58cebe39..4e3457b23d7db62afd28984584fd4cc6efe47089 100644
--- a/OTBConfig.cmake.in
+++ b/OTBConfig.cmake.in
@@ -109,6 +109,7 @@ SET(OTB_GL_USE_ACCEL "@OTB_GL_USE_ACCEL@")
# Curl library
SET(OTB_USE_CURL "@OTB_USE_CURL@")
+SET(OTB_CURL_MULTI_AVAILABLE "@OTB_CURL_MULTI_AVAILABLE@")
#Developpers message
SET(OTB_SHOW_ALL_MSG_DEBUG "@OTB_SHOW_ALL_MSG_DEBUG@")
diff --git a/UseOTB.cmake.in b/UseOTB.cmake.in
index e4e15b41c3e9d8990f5722862255e1b78ef6790b..7dac912f8aff53aec5060b35ceb86960d1ac25b9 100644
--- a/UseOTB.cmake.in
+++ b/UseOTB.cmake.in
@@ -132,3 +132,7 @@ ENDIF(OTB_USE_EXTERNAL_GDAL)
IF(OTB_USE_CURL)
ADD_DEFINITIONS(-DOTB_USE_CURL)
ENDIF(OTB_USE_CURL)
+
+IF(OTB_CURL_MULTI_AVAILABLE)
+ADD_DEFINITIONS(-DOTB_CURL_MULTI_AVAILABLE)
+ENDIF(OTB_CURL_MULTI_AVAILABLE)