Commit e9f7d2b5 authored by Emmanuel Christophe's avatar Emmanuel Christophe
Browse files

ENH: TileMapImageIO uses CurlHelper, fallback in CurlHelper when curl multi is not available

parent f2584a08
......@@ -366,6 +366,10 @@ IF(OTB_USE_CURL)
INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
# TODO add test to check if curl multi is available
OPTION(OTB_CURL_MULTI_AVAILABLE "Whether or not the curl has curlmulti capabilities" ON)
MARK_AS_ADVANCED(OTB_CURL_MULTI_AVAILABLE)
ADD_DEFINITIONS(-DOTB_CURL_MULTI_AVAILABLE)
ENDIF(OTB_USE_CURL)
#-------------------------------
......
......@@ -17,7 +17,7 @@
=========================================================================*/
#include "otbCurlHelper.h"
#include "otbMacro.h"
#ifdef OTB_USE_CURL
#include <curl/curl.h>
......@@ -58,6 +58,11 @@ int CurlHelper::TestUrlAvailability(const std::string& url) const
}
int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string filename) const
{
return RetrieveFile(urlStream.str(), filename);
}
int CurlHelper::RetrieveFile(const std::string& urlString, std::string filename) const
{
#ifdef OTB_USE_CURL
CURL * curl;
......@@ -67,7 +72,7 @@ int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string fi
curl = curl_easy_init();
char url[256];
strcpy(url, urlStream.str().data());
strcpy(url, urlString.data());
// std::cout << url << std::endl;
if (curl)
......@@ -92,5 +97,183 @@ int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string fi
#endif
}
int CurlHelper::RetrieveFileMulti(const std::vector<std::string>& listURLs,
const std::vector<std::string>& listFilename,
int maxConnect) const
{
#ifdef OTB_USE_CURL
#ifdef OTB_CURL_MULTI_AVAILABLE
otbMsgDevMacro(<< "Using curl multi");
std::string m_Browser = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11";
CURLM * multiHandle;
std::vector<CURL *> listCurlHandles;
std::vector<FILE *> listFiles;
std::vector<std::string>::const_iterator filename;
filename = listFilename.begin();
while (filename != listFilename.end() )
{
FILE* lOutputFile = fopen((*filename).c_str(), "w");
if (lOutputFile == NULL)
{
itkExceptionMacro(<< "otbCurlHelper: bad file name: " << (*filename).c_str());
}
// Add file to vector
listFiles.push_back(lOutputFile);
++filename;
}
// Initialize curl multi handle
multiHandle = curl_multi_init();
if (!multiHandle)
{
itkExceptionMacro(<< "otbCurlHelper: Curl multi handle init error.");
}
listCurlHandles.clear();
// Initialize curl handle
std::vector<std::string>::const_iterator url;
std::vector<FILE *>::const_iterator file;
url = listURLs.begin();
file = listFiles.begin();
while ( (url != listURLs.end()) && (file != listFiles.end() ))
{
CURL * lEasyHandle;
lEasyHandle = curl_easy_init();
if (!lEasyHandle)
{
itkExceptionMacro(<< "otbCurlHelper: Curl easy handle init error.");
}
// Param easy handle
curl_easy_setopt(lEasyHandle, CURLOPT_USERAGENT, m_Browser.data());
curl_easy_setopt(lEasyHandle, CURLOPT_URL, (*url).data());
curl_easy_setopt(lEasyHandle, CURLOPT_WRITEDATA, *file);
// Add easy handle to multi handle
curl_multi_add_handle(multiHandle, lEasyHandle);
// Add hanle to vector
listCurlHandles.push_back(lEasyHandle);
++url;
++file;
}
//fetch tiles
// Configure multi handle - set the maximum connections
curl_multi_setopt(multiHandle, CURLMOPT_MAXCONNECTS, maxConnect);
curl_multi_setopt(multiHandle, CURLMOPT_PIPELINING, 0);
// Perform
int lStillRunning;
while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle, &lStillRunning));
// Now get that URL
while (lStillRunning)
{
struct timeval timeout;
int rc; // Return code
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to play around with */
timeout.tv_sec = 0;
timeout.tv_usec = 1;
/* get file descriptors from the transfers */
curl_multi_fdset(multiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
switch (rc)
{
case -1:
/* select error */
break;
case 0:
/* timeout */
default:
/* timeout or readable/writable sockets */
while (
CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle, &lStillRunning)
)
;
break;
}
}
int remaining_msgs = 1;
int error = 0;
CURLMsg *msg;
while (remaining_msgs)
{
msg = curl_multi_info_read(multiHandle, &remaining_msgs);
if (msg != NULL)
{
if (CURLE_OK != msg->data.result) error = 1;
}
}
if (error != 0)
{
itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
}
// Cleanup
// Close files
for (unsigned int currentFile = 0; currentFile < listFiles.size(); currentFile++)
{
fclose(listFiles[currentFile]);
}
listFiles.clear();
// Cleanup easy handles
for (unsigned int currentHandle = 0; currentHandle < listCurlHandles.size(); currentHandle++)
{
curl_easy_cleanup(listCurlHandles[currentHandle]);
}
listCurlHandles.clear();
// Cleanup multi handle
curl_multi_cleanup(multiHandle);
#else
//fallback on non curl multi
otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");
std::vector<std::string>::const_iterator url;
std::vector<FILE *>::const_iterator file;
url = listURLs.begin();
file = listFiles.begin();
while ( (url != listURLs.end()) && (file != listFiles.end() ))
{
RetrieveFile(*url, *file);
}
#endif
#else
otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
return -1;
#endif
}
}
......@@ -43,7 +43,11 @@ public:
int TestUrlAvailability(const std::string& url) const;
int RetrieveFile(const std::ostringstream& urlStream, std::string filename) const;
int RetrieveFile(const std::string& urlString, std::string filename) const;
int RetrieveFileMulti(const std::vector<std::string>& listURLs,
const std::vector<std::string>& listFiles,
int maxConnect) const;
protected:
CurlHelper() {}
~CurlHelper() {}
......
......@@ -38,6 +38,8 @@
#include "base/ossimFilename.h"
#include "itkTimeProbe.h"
#include "otbCurlHelper.h"
namespace otb
{
......@@ -78,8 +80,6 @@ TileMapImageIO::TileMapImageIO()
m_FileNameIsServerName = false;
// Initialize browser
m_Browser = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11";
// Set maximum of connections to 10
m_MaxConnect = 10;
......@@ -152,17 +152,9 @@ void TileMapImageIO::Read(void* buffer)
int nTilesX = (int) ceil(totSamples / 256.) + 1;
int nTilesY = (int) ceil(totLines / 256.) + 1;
// Initialize curl multi handle
m_MultiHandle = curl_multi_init();
if (!m_MultiHandle)
{
itkExceptionMacro(<< "Tile Map IO : Curl mutli handle init error.");
}
// Clear vectors
m_ListCurlHandles.clear();
m_ListFiles.clear();
m_ListFilename.clear();
m_ListURLs.clear();
m_ListTiles.clear();
......@@ -182,17 +174,16 @@ void TileMapImageIO::Read(void* buffer)
// Try to read tile from cache
if (!this->CanReadFromCache(m_ListTiles.back().filename))
{
// Generate curl handle for this tile
this->GenerateCURLHandle(m_ListTiles.back());
this->GenerateURL(m_ListTiles.back().x, m_ListTiles.back().y);
m_ListFilename.push_back(m_ListTiles.back().filename);
}
}
}
// Fetch tiles from net
this->FetchTiles();
CurlHelper::Pointer curlHelper = CurlHelper::New();
curlHelper->RetrieveFileMulti(m_ListURLs, m_ListFilename, m_MaxConnect);
// Cleanup datas use to download tiles
this->Cleanup();
m_ListURLs.clear();
// Generate buffer
this->GenerateBuffer(p);
......@@ -248,48 +239,6 @@ bool TileMapImageIO::CanReadFromCache(std::string filename)
return lCanRead;
}
/*
* This method generate curl handles and add to multi handle
*/
void TileMapImageIO::GenerateCURLHandle(TileNameAndCoordType tileInfo)
{
// Generate URL
this->GenerateURL(tileInfo.x, tileInfo.y);
// Create file
FILE* lOutputFile = fopen(tileInfo.filename.c_str(), "w");
if (lOutputFile == NULL)
{
itkExceptionMacro(<< "TileMap read : bad file name.");
}
// Add file to vector
m_ListFiles.push_back(lOutputFile);
// Initialize curl handle
CURL * lEasyHandle;
lEasyHandle = curl_easy_init();
if (!lEasyHandle)
{
itkExceptionMacro(<< "Tile Map IO : Curl easy handle init error.");
}
// Param easy handle
curl_easy_setopt(lEasyHandle, CURLOPT_USERAGENT, m_Browser.data());
curl_easy_setopt(lEasyHandle, CURLOPT_URL, m_ListURLs.back().data());
curl_easy_setopt(lEasyHandle, CURLOPT_WRITEDATA, m_ListFiles.back());
// Add easy handle to multi handle
curl_multi_add_handle(m_MultiHandle, lEasyHandle);
// Add hanle to vector
m_ListCurlHandles.push_back(lEasyHandle);
}
/*
* This method generate URLs
*/
......@@ -347,108 +296,6 @@ void TileMapImageIO::GenerateURL(double x, double y)
m_ListURLs.push_back(urlStream.str());
}
/*
* This method perform curl multi handle
*/
void TileMapImageIO::FetchTiles()
{
// Configure multi handle - set the maximum connections
curl_multi_setopt(m_MultiHandle, CURLMOPT_MAXCONNECTS, m_MaxConnect);
curl_multi_setopt(m_MultiHandle, CURLMOPT_PIPELINING, 0);
// Perform
int lStillRunning;
while (
CURLM_CALL_MULTI_PERFORM == curl_multi_perform(m_MultiHandle, &lStillRunning)
)
;
// Now get that URL
while (lStillRunning)
{
struct timeval timeout;
int rc; // Return code
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to play around with */
timeout.tv_sec = 0;
timeout.tv_usec = 1;
/* get file descriptors from the transfers */
curl_multi_fdset(m_MultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
switch (rc)
{
case -1:
/* select error */
break;
case 0:
/* timeout */
default:
/* timeout or readable/writable sockets */
while (
CURLM_CALL_MULTI_PERFORM == curl_multi_perform(m_MultiHandle, &lStillRunning)
)
;
break;
}
}
int remaining_msgs = 1;
int error = 0;
CURLMsg *msg;
while (remaining_msgs)
{
msg = curl_multi_info_read(m_MultiHandle, &remaining_msgs);
if (msg != NULL)
{
if (CURLE_OK != msg->data.result) error = 1;
}
}
if (error != 0)
{
itkExceptionMacro(<< "TileMapImageIO : Error occurs while perform Multi handle");
}
}
/*
* This method cleanup datas
*/
void TileMapImageIO::Cleanup()
{
// Close files
for (unsigned int currentFile = 0; currentFile < m_ListFiles.size(); currentFile++)
{
fclose(m_ListFiles[currentFile]);
}
m_ListFiles.clear();
// Cleanup easy handles
for (unsigned int currentHandle = 0; currentHandle < m_ListCurlHandles.size(); currentHandle++)
{
curl_easy_cleanup(m_ListCurlHandles[currentHandle]);
}
m_ListCurlHandles.clear();
// Cleanup multi handle
curl_multi_cleanup(m_MultiHandle);
// Cleanup url vector
m_ListURLs.clear();
}
/*
* This method generate the output buffer
*/
......
......@@ -30,9 +30,6 @@
#include "itkImageIOBase.h"
#include "otbImageRegionTileMapSplitter.h"
/* Curl Library*/
#include <curl/curl.h>
namespace otb
{
......@@ -190,19 +187,13 @@ private:
/** CURL Multi */
void GenerateTileInfo(double x, double y, int numTileX, int numTileY);
bool CanReadFromCache(std::string filename);
void GenerateCURLHandle(TileNameAndCoordType tileInfo);
void GenerateURL(double x, double y);
void FetchTiles();
void Cleanup();
void GenerateBuffer(unsigned char * p);
void ReadTile(std::string filename, void * buffer);
CURLM * m_MultiHandle;
std::vector<CURL *> m_ListCurlHandles;
std::vector<FILE *> m_ListFiles;
std::vector<std::string> m_ListFilename;
std::vector<std::string> m_ListURLs;
std::vector<TileNameAndCoordType> m_ListTiles;
std::string m_Browser;
int m_MaxConnect;
/** Byte per pixel pixel */
......
......@@ -109,6 +109,7 @@ SET(OTB_GL_USE_ACCEL "@OTB_GL_USE_ACCEL@")
# Curl library
SET(OTB_USE_CURL "@OTB_USE_CURL@")
SET(OTB_CURL_MULTI_AVAILABLE "@OTB_CURL_MULTI_AVAILABLE@")
#Developpers message
SET(OTB_SHOW_ALL_MSG_DEBUG "@OTB_SHOW_ALL_MSG_DEBUG@")
......
......@@ -132,3 +132,7 @@ ENDIF(OTB_USE_EXTERNAL_GDAL)
IF(OTB_USE_CURL)
ADD_DEFINITIONS(-DOTB_USE_CURL)
ENDIF(OTB_USE_CURL)
IF(OTB_CURL_MULTI_AVAILABLE)
ADD_DEFINITIONS(-DOTB_CURL_MULTI_AVAILABLE)
ENDIF(OTB_CURL_MULTI_AVAILABLE)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment