otbCurlHelper.cxx 15.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*=========================================================================

  Program:   ORFEO Toolbox
  Language:  C++
  Date:      $Date$
  Version:   $Revision$


  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See OTBCopyright.txt for details.


     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

19
#include <cstdio>
Julien Malik's avatar
Julien Malik committed
20
#include <cstring>
21
#include <sstream>
22
#include <itkLightObject.h>
23
#include "otbConfigure.h" // for OTB_USE_CURL
24
#include "otbMacro.h"
25
#include "otbCurlHelper.h"
26

27 28 29
#ifdef OTB_USE_CURL
#  include "otb_curl.h"
#endif
30

31 32 33 34
namespace otb
{

#ifdef OTB_USE_CURL
35 36

/**
37
 * Class to handle the CURLcode and CURLMCode and throw exceptions when needed
38
 */
39
class CurlHandleError {
40

41 42 43 44 45 46 47 48 49
public:
  /** Processing CURLcode */
  static void ProcessCURLcode(CURLcode curlCode)
  {
    if(curlCode != CURLE_OK)
       {
       itkGenericExceptionMacro(<<" Curl Error : "<< curl_easy_strerror(curlCode));
       }
  }
50

51 52 53 54 55 56 57 58 59
  /** Processing CURLMcode */
  static void ProcessCURLcode(CURLMcode curlMCode)
  {
    if(curlMCode != CURLM_OK)
      {
      itkGenericExceptionMacro(<<" CurlM Error : "<< curl_multi_strerror(curlMCode));
      }
  }
};
60

61 62 63 64
/**
 * Resource class that create and clean the curl environment proprely
 * in case of a thrown exception
 */
65
class CurlResource : public itk::LightObject {
66 67

public:
68 69 70 71
  /** Standard class typedefs. */
  typedef CurlResource                  Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
72
  typedef itk::LightObject              Superclass;
73 74 75

  itkTypeMacro(CurlResource, itk::LightObject);
  itkNewMacro(Self);
76

77 78 79 80 81 82 83
  /** Get the curl object */
  CURL * GetCurlResource()
  {
    return m_Curl;
  }

protected:
84 85 86
  CurlResource()
  {
    m_Curl = curl_easy_init();
87

OTB Bot's avatar
STYLE  
OTB Bot committed
88
    if (!m_Curl)
89 90 91 92
      {
      itkExceptionMacro(<<" otbCurlHelper::CurlResource Curl handle init error.");
      }
  }
93

94
  ~CurlResource() ITK_OVERRIDE
95 96 97 98 99 100 101 102 103 104 105
  {
    curl_easy_cleanup(m_Curl);
  }

private:
  CURL * m_Curl;
  // prevent copying and assignment; not implemented
  CurlResource (const CurlResource &);
  CurlResource & operator= (const CurlResource &);
};  //end of class CurlResource

106 107

#ifdef OTB_CURL_MULTI_AVAILABLE
108 109 110 111
/**
 * Resource class that create and clean the curl multi environment
 * proprely in case of a thrown exception
 */
112
class CurlMultiResource : public itk::LightObject {
113 114

public:
115 116 117 118
  /** Standard class typedefs. */
  typedef CurlMultiResource             Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
119
  typedef itk::LightObject              Superclass;
120 121 122

  itkTypeMacro(CurlMultiResource, itk::LightObject);
  itkNewMacro(Self);
123

124 125 126 127 128 129 130
  CURLM * GetCurlMultiResource()
  {
    return m_Curl;
  }

protected:
  CurlMultiResource()
131 132
  {
    m_Curl = curl_multi_init();
133

OTB Bot's avatar
STYLE  
OTB Bot committed
134
    if (!m_Curl)
135 136 137 138 139
      {
      itkExceptionMacro(<<" otbCurlHelper::CurlMultiResource Curl multi handle init error.");
      }
  }

140
  ~CurlMultiResource() ITK_OVERRIDE
141 142 143
  {
    curl_multi_cleanup(m_Curl);
  }
144

145 146 147 148 149 150
private:
  CURLM * m_Curl;
  // prevent copying and assignment; not implemented
  CurlMultiResource (const CurlMultiResource &);
  CurlMultiResource & operator= (const CurlMultiResource &);
}; //end of class CurlMultiResource
151
#endif // OTB_CURL_MULTI_AVAILABLE
152 153 154 155 156

/**
 * Resource class that create FILE * and  close the FILE *  descriptor
 * proprely in case of a thrown exception
 */
157
class CurlFileDescriptorResource : public itk::LightObject {
158 159

public:
160 161 162 163 164 165 166 167
  /** Standard class typedefs. */
  typedef CurlFileDescriptorResource    Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
  typedef itk::LightObject              Superclass;

  itkTypeMacro(CurlFileDescriptorResource, itk::LightObject);
  itkNewMacro(Self);
168

169 170 171 172 173 174
  FILE * GetFileResource()
  {
    return m_File;
  }

  void OpenFile(const char* infname)
175 176
  {
    m_File = fopen(infname, "wb");
177

OTB Bot's avatar
STYLE  
OTB Bot committed
178
    if (m_File == NULL)
179 180 181 182
      {
      itkExceptionMacro(<<" otbCurlHelper::FileResource : failed to open the file ."<< infname);
      }
  }
183

184 185
protected:
  CurlFileDescriptorResource(){}
186

187
  ~CurlFileDescriptorResource() ITK_OVERRIDE
188
  {
189
    fclose(m_File);
190
  }
191

192
private:
193
  FILE *      m_File;
194

195
  // prevent copying and assignment; not implemented
196 197
  CurlFileDescriptorResource (const CurlFileDescriptorResource &);
  CurlFileDescriptorResource & operator= (const CurlFileDescriptorResource &);
198 199
}; //end of class FileResource

200
#endif // OTB_USE_CURL
201

202 203
bool CurlHelper::TestUrlAvailability(const std::string& url) const
{
204 205
  (void)url;

206 207 208
#ifdef OTB_USE_CURL
  // Set up a curl resource
  CurlResource::Pointer curlResource = CurlResource::New();
209 210 211

  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
212
  // Set the dummy write function
213
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
214
                               &Self::CallbackWriteDataDummy));
215
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
216

217 218
  // Perform requet
  CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
219

220 221 222
  otbMsgDevMacro(<< "CurlHelper::TestUrlAvailability : curl_easy_perform returned "
                 << easyPerformResult << " --> "
                 << curl_easy_strerror(easyPerformResult));
223

224 225 226 227 228 229
  // Check the curl_easy_perform return code : actually we tests only
  // the availability of the url, so if the file requested size is less
  // than 1 byte, and the url is valid it returns CURLE_OK, or if the
  // url is valid and the file size is greater than 1 byte it returns
  // the error CURLE_FILESIZE_EXCEEDED
  if ( easyPerformResult == CURLE_OK || easyPerformResult == CURLE_FILESIZE_EXCEEDED )
230
    {
231
    return true;
232
    }
233

234
  return false;
235 236 237 238
#else
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
  return false;
#endif
239 240
}

241 242
bool CurlHelper::IsCurlReturnHttpError(const std::string& url) const
{
243 244 245 246
#ifdef OTB_USE_CURL
  // Set up a curl resource
  CurlResource::Pointer curlResource = CurlResource::New();

247 248
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
249
  // Set the dummy write function
250
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
251
                               &Self::CallbackWriteDataDummy));
252 253
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_FAILONERROR, 1));
254 255 256 257

  // Perform requet
  CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
  if ( easyPerformResult == CURLE_HTTP_RETURNED_ERROR )
258
    {
259
    return true;
260
    }
261

262
  return false;
263
#else
264
  (void)url;
265 266 267
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
  return false;
#endif
268 269
}

270 271
int CurlHelper::RetrieveUrlInMemory(const std::string& url, std::string& output) const
{
272 273 274 275
#ifdef OTB_USE_CURL
  otbMsgDevMacro(<< "Retrieving: " << url);
  CURLcode res = CURLE_OK;
  CurlResource::Pointer curlResource = CurlResource::New();
276

277
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.c_str()));
278

279 280
  // Settimeout
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, m_Timeout));
281

282 283
  // Use our writing static function to avoid file descriptor
  // pointer crash on windows
284
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
285
                               &Self::CallbackWriteDataToStringStream));
286

287 288
  // Say the file where to write the received data
  std::ostringstream* outputStream = new std::ostringstream;
289
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA, (void*) outputStream));
290

291
  // Perform request
292
  CurlHandleError::ProcessCURLcode(curl_easy_perform(curlResource->GetCurlResource()));
293

294 295 296
  // Save output
  output = outputStream->str();
  otbMsgDevMacro("curl output : " << output);
297

298 299
  // Clean up
  delete outputStream;
300

301 302 303
  otbMsgDevMacro(<< " -> " << res);
  return res;
#else
304 305
  (void)url;
  (void)output;
306
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
307
  return -1;
308
#endif
309 310
}

311
int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string filename) const
312 313 314 315 316
{
  return RetrieveFile(urlStream.str(), filename);
}

int CurlHelper::RetrieveFile(const std::string& urlString, std::string filename) const
317
{
OTB Bot's avatar
STYLE  
OTB Bot committed
318
#ifdef OTB_USE_CURL
319
  otbMsgDevMacro(<< "Retrieving: " << urlString);
320

321
  CURLcode res = CURLE_OK;
322

323
  CurlResource::Pointer curlResource = CurlResource::New();
OTB Bot's avatar
STYLE  
OTB Bot committed
324

325 326
  CurlFileDescriptorResource::Pointer output_file = CurlFileDescriptorResource::New();
  output_file->OpenFile(filename.c_str());
327

328 329
  char url[256];
  strcpy(url, urlString.c_str());
330

331
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url));
332

333 334
  // Set timeout
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, m_Timeout));
OTB Bot's avatar
STYLE  
OTB Bot committed
335

336 337
  // Use our writing static function to avoid file descriptor
  // pointer crash on windows
338
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
339 340
                               &Self::CallbackWriteDataToFile));
  // Say the file where to write the received data
341
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA,
342
                               (void*) output_file->GetFileResource()));
OTB Bot's avatar
STYLE  
OTB Bot committed
343

344
  CurlHandleError::ProcessCURLcode(curl_easy_perform(curlResource->GetCurlResource()));
345

346 347 348
  otbMsgDevMacro(<< " -> " << res);
  return res;
#else
349 350
  (void)urlString;
  (void)filename;
351
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
352
  return -1;
353
#endif
354 355
}

OTB Bot's avatar
STYLE  
OTB Bot committed
356
int CurlHelper::RetrieveFileMulti(const std::vector<std::string>& listURLs,
357
                                  const std::vector<std::string>& listFilename,
358
                                  int maxConnect) const
359
{
360

361
#ifdef OTB_USE_CURL
362 363
#if 0
//#ifdef OTB_CURL_MULTI_AVAILABLE
364
  otbMsgDevMacro(<< "Using curl multi");
365

366 367
  // Initialize curl handle resource
  CurlMultiResource::Pointer  multiHandle = CurlMultiResource::New();
368

369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
  std::vector<CurlResource::Pointer> listCurlHandles;
  std::vector<CurlFileDescriptorResource::Pointer> listFiles;

  std::vector<std::string>::const_iterator filename;
  filename = listFilename.begin();
  while (filename != listFilename.end())
    {
    CurlFileDescriptorResource::Pointer lOutputFile = CurlFileDescriptorResource::New();
    lOutputFile->OpenFile((*filename).c_str());

    // Add file to vector
    listFiles.push_back(lOutputFile);
    ++filename;
    }

  listCurlHandles.clear();

  std::vector<std::string>::const_iterator url;
  std::vector<CurlFileDescriptorResource::Pointer>::const_iterator      file;
  url = listURLs.begin();
  file = listFiles.begin();
  while ((url != listURLs.end()) && (file != listFiles.end()))
    {
    otbMsgDevMacro(<< "Retrieving: " << (*url).data());
    CurlResource::Pointer lEasyHandle = CurlResource::New();

    // Param easy handle
396 397 398
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_URL, (*url).data()));
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEFUNCTION,
399
                                 &Self::CallbackWriteDataToFile));
400
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEDATA,
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
401
                                 (void*) (*file)->GetFileResource()));
402

403
    // Add easy handle to multi handle
404
    CurlHandleError::ProcessCURLcode(curl_multi_add_handle(multiHandle->GetCurlMultiResource(), lEasyHandle->GetCurlResource()));
405 406 407 408 409 410 411 412 413

    // Add hanle to vector
    listCurlHandles.push_back(lEasyHandle);
    ++url;
    ++file;
    }

  //fetch tiles
  // Configure multi handle - set the maximum connections
414 415
  CurlHandleError::ProcessCURLcode(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_MAXCONNECTS, maxConnect));
  CurlHandleError::ProcessCURLcode(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_PIPELINING, 0));
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441

  // Perform
  int lStillRunning;

  while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));

  // Now get that URL
  while (lStillRunning)
    {
    struct timeval timeout;
    int            rc; // Return code

    fd_set fdread;
    fd_set fdwrite;
    fd_set fdexcep;
    int    maxfd;

    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* set a suitable timeout to play around with */
    timeout.tv_sec = 0;
    timeout.tv_usec = 1;

    /* get file descriptors from the transfers */
442
    CurlHandleError::ProcessCURLcode(curl_multi_fdset(multiHandle->GetCurlMultiResource(), &fdread, &fdwrite, &fdexcep, &maxfd));
443 444 445 446 447 448 449 450 451 452 453 454 455 456

    rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);

    switch (rc)
      {
      case -1:
        /* select error */
        break;
      case 0:
        /* timeout */
      default:
        /* timeout or readable/writable sockets */
        while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
        break;
457
      }
458 459 460 461 462 463 464 465 466
    }

  int      remaining_msgs = 1;
  int      error = 0;
  CURLMsg *msg;
  while (remaining_msgs)
    {
    msg = curl_multi_info_read(multiHandle->GetCurlMultiResource(), &remaining_msgs);
    if (msg != NULL)
467
      {
468 469 470 471 472
      if (CURLE_OK != msg->data.result) error = 1;
      }
    }

  if (error != 0)
473
    {
474
    itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
475
    }
476 477 478 479

  // Cleanup
  listFiles.clear();
  listCurlHandles.clear();
Julien Malik's avatar
Julien Malik committed
480

OTB Bot's avatar
STYLE  
OTB Bot committed
481
  return 0;
482
#else
Guillaume Pasero's avatar
Guillaume Pasero committed
483
  (void)maxConnect;
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
  //fallback on non curl multi
  otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");

  std::vector<std::string>::const_iterator url;
  std::vector<std::string>::const_iterator file;
  url = listURLs.begin();
  file = listFilename.begin();
  int res = 0;
  int resTmp = -1;
  while ((url != listURLs.end()) && (file != listFilename.end()))
    {
    resTmp = RetrieveFile(*url, *file);
    if (res == 0) res = resTmp;
    ++url;
    ++file;
    }

  return res;
#endif
#else
504 505 506
  (void)maxConnect;
  (void)listURLs;
  (void)listFilename;
507
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
508
  return -1;
509
#endif
510 511
}

512
size_t CurlHelper::CallbackWriteDataToFile(void* ptr, size_t size, size_t nmemb, void* data)
513 514 515
{
  size_t written;

OTB Bot's avatar
STYLE  
OTB Bot committed
516
  FILE * fDescriptor = (FILE *) (data);
517

OTB Bot's avatar
STYLE  
OTB Bot committed
518
  written = fwrite(ptr, size, nmemb, fDescriptor);
519 520 521 522

  return written;
}

523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
/*
size_t CurlHelper::CallbackWriteDataToCharVector(void *ptr, size_t size, size_t nmemb, void *data)
{
  register int realsize = (int)(size * nmemb);

  std::vector<char> *vec
    = static_cast<std::vector<char>*>(data);
  const char* chPtr = static_cast<char*>(ptr);
  vec->insert(vec->end(), chPtr, chPtr + realsize);

  return realsize;
}
*/

size_t CurlHelper::CallbackWriteDataToStringStream(void *ptr, size_t size, size_t nmemb, void *data)
{
  std::ostringstream& stream = *reinterpret_cast<std::ostringstream*>(data);
  stream << reinterpret_cast<char*>(ptr);
  return size * nmemb;
}

544
size_t CurlHelper::CallbackWriteDataDummy(void *itkNotUsed(ptr), size_t size, size_t nmemb, void *itkNotUsed(data))
545 546 547 548
{
  return size * nmemb;
}

549
}