otbCurlHelper.cxx 16.3 KB
Newer Older
1
/*
Julien Michel's avatar
Julien Michel committed
2
 * Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 * This file is part of Orfeo Toolbox
 *
 *     https://www.orfeo-toolbox.org/
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
20 21


22
#include <cstdio>
Julien Malik's avatar
Julien Malik committed
23
#include <cstring>
24
#include <sstream>
25
#include <itkLightObject.h>
26
#include "otbConfigure.h" // for OTB_USE_CURL
27
#include "otbMacro.h"
28
#include "otbCurlHelper.h"
29

30 31 32
#ifdef OTB_USE_CURL
#  include "otb_curl.h"
#endif
33

34 35 36 37
namespace otb
{

#ifdef OTB_USE_CURL
38 39

/**
40
 * Class to handle the CURLcode and CURLMCode and throw exceptions when needed
41
 */
42
class CurlHandleError {
43

44 45 46 47 48 49 50 51 52
public:
  /** Processing CURLcode */
  static void ProcessCURLcode(CURLcode curlCode)
  {
    if(curlCode != CURLE_OK)
       {
       itkGenericExceptionMacro(<<" Curl Error : "<< curl_easy_strerror(curlCode));
       }
  }
53

54 55 56 57 58 59 60 61 62
  /** Processing CURLMcode */
  static void ProcessCURLcode(CURLMcode curlMCode)
  {
    if(curlMCode != CURLM_OK)
      {
      itkGenericExceptionMacro(<<" CurlM Error : "<< curl_multi_strerror(curlMCode));
      }
  }
};
63

64 65 66 67
/**
 * Resource class that create and clean the curl environment proprely
 * in case of a thrown exception
 */
68
class CurlResource : public itk::LightObject {
69 70

public:
71 72 73 74
  /** Standard class typedefs. */
  typedef CurlResource                  Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
75
  typedef itk::LightObject              Superclass;
76 77 78

  itkTypeMacro(CurlResource, itk::LightObject);
  itkNewMacro(Self);
79

80 81 82 83 84 85 86
  /** Get the curl object */
  CURL * GetCurlResource()
  {
    return m_Curl;
  }

protected:
87 88 89
  CurlResource()
  {
    m_Curl = curl_easy_init();
90

OTB Bot's avatar
STYLE  
OTB Bot committed
91
    if (!m_Curl)
92 93 94 95
      {
      itkExceptionMacro(<<" otbCurlHelper::CurlResource Curl handle init error.");
      }
  }
96

97
  ~CurlResource() override
98 99 100 101 102 103 104 105 106 107 108
  {
    curl_easy_cleanup(m_Curl);
  }

private:
  CURL * m_Curl;
  // prevent copying and assignment; not implemented
  CurlResource (const CurlResource &);
  CurlResource & operator= (const CurlResource &);
};  //end of class CurlResource

109 110

#ifdef OTB_CURL_MULTI_AVAILABLE
111 112 113 114
/**
 * Resource class that create and clean the curl multi environment
 * proprely in case of a thrown exception
 */
115
class CurlMultiResource : public itk::LightObject {
116 117

public:
118 119 120 121
  /** Standard class typedefs. */
  typedef CurlMultiResource             Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
122
  typedef itk::LightObject              Superclass;
123 124 125

  itkTypeMacro(CurlMultiResource, itk::LightObject);
  itkNewMacro(Self);
126

127 128 129 130 131 132 133
  CURLM * GetCurlMultiResource()
  {
    return m_Curl;
  }

protected:
  CurlMultiResource()
134 135
  {
    m_Curl = curl_multi_init();
136

OTB Bot's avatar
STYLE  
OTB Bot committed
137
    if (!m_Curl)
138 139 140 141 142
      {
      itkExceptionMacro(<<" otbCurlHelper::CurlMultiResource Curl multi handle init error.");
      }
  }

143
  ~CurlMultiResource() override
144 145 146
  {
    curl_multi_cleanup(m_Curl);
  }
147

148 149 150 151 152 153
private:
  CURLM * m_Curl;
  // prevent copying and assignment; not implemented
  CurlMultiResource (const CurlMultiResource &);
  CurlMultiResource & operator= (const CurlMultiResource &);
}; //end of class CurlMultiResource
154
#endif // OTB_CURL_MULTI_AVAILABLE
155 156 157 158 159

/**
 * Resource class that create FILE * and  close the FILE *  descriptor
 * proprely in case of a thrown exception
 */
160
class CurlFileDescriptorResource : public itk::LightObject {
161 162

public:
163 164 165 166 167 168 169 170
  /** Standard class typedefs. */
  typedef CurlFileDescriptorResource    Self;
  typedef itk::SmartPointer<Self>       Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;
  typedef itk::LightObject              Superclass;

  itkTypeMacro(CurlFileDescriptorResource, itk::LightObject);
  itkNewMacro(Self);
171

172 173 174 175 176 177
  FILE * GetFileResource()
  {
    return m_File;
  }

  void OpenFile(const char* infname)
178 179
  {
    m_File = fopen(infname, "wb");
180

181
    if (m_File == nullptr)
182 183 184 185
      {
      itkExceptionMacro(<<" otbCurlHelper::FileResource : failed to open the file ."<< infname);
      }
  }
186

187
protected:
188
  CurlFileDescriptorResource(): m_File(nullptr) {}
189

190
  ~CurlFileDescriptorResource() override
191
  {
192
    fclose(m_File);
193
  }
194

195
private:
196
  FILE *      m_File;
197

198 199 200
  // prevent copying and assignment
  CurlFileDescriptorResource (const CurlFileDescriptorResource &) = delete;
  CurlFileDescriptorResource & operator= (const CurlFileDescriptorResource &) = delete;
201 202
}; //end of class FileResource

203
#endif // OTB_USE_CURL
204

205 206
bool CurlHelper::TestUrlAvailability(const std::string& url) const
{
207 208
  (void)url;

209 210 211
#ifdef OTB_USE_CURL
  // Set up a curl resource
  CurlResource::Pointer curlResource = CurlResource::New();
212 213 214

  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
215
  // Set the dummy write function
216
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
217
                               &Self::CallbackWriteDataDummy));
218
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
219

220 221
  // Perform requet
  CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
222

223 224 225
  otbMsgDevMacro(<< "CurlHelper::TestUrlAvailability : curl_easy_perform returned "
                 << easyPerformResult << " --> "
                 << curl_easy_strerror(easyPerformResult));
226

227 228 229 230 231 232
  // Check the curl_easy_perform return code : actually we tests only
  // the availability of the url, so if the file requested size is less
  // than 1 byte, and the url is valid it returns CURLE_OK, or if the
  // url is valid and the file size is greater than 1 byte it returns
  // the error CURLE_FILESIZE_EXCEEDED
  if ( easyPerformResult == CURLE_OK || easyPerformResult == CURLE_FILESIZE_EXCEEDED )
233
    {
234
    return true;
235
    }
236

237
  return false;
238 239 240 241
#else
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
  return false;
#endif
242 243
}

244 245
bool CurlHelper::IsCurlReturnHttpError(const std::string& url) const
{
246 247 248 249
#ifdef OTB_USE_CURL
  // Set up a curl resource
  CurlResource::Pointer curlResource = CurlResource::New();

250 251
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.data()));
252
  // Set the dummy write function
253
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
254
                               &Self::CallbackWriteDataDummy));
255 256
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_MAXFILESIZE, 1));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_FAILONERROR, 1));
257 258 259 260

  // Perform requet
  CURLcode easyPerformResult = curl_easy_perform(curlResource->GetCurlResource());
  if ( easyPerformResult == CURLE_HTTP_RETURNED_ERROR )
261
    {
262
    return true;
263
    }
264

265
  return false;
266
#else
267
  (void)url;
268 269 270
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
  return false;
#endif
271 272
}

273 274
int CurlHelper::RetrieveUrlInMemory(const std::string& url, std::string& output) const
{
275 276 277 278
#ifdef OTB_USE_CURL
  otbMsgDevMacro(<< "Retrieving: " << url);
  CURLcode res = CURLE_OK;
  CurlResource::Pointer curlResource = CurlResource::New();
279

280
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url.c_str()));
281

282 283
  // Settimeout
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, m_Timeout));
284

285 286
  // Use our writing static function to avoid file descriptor
  // pointer crash on windows
287
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
288
                               &Self::CallbackWriteDataToStringStream));
289

290 291
  // Say the file where to write the received data
  std::ostringstream* outputStream = new std::ostringstream;
292
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA, (void*) outputStream));
293

294
  // Perform request
295
  CurlHandleError::ProcessCURLcode(curl_easy_perform(curlResource->GetCurlResource()));
296

297 298 299
  // Save output
  output = outputStream->str();
  otbMsgDevMacro("curl output : " << output);
300

301 302
  // Clean up
  delete outputStream;
303

304 305 306
  otbMsgDevMacro(<< " -> " << res);
  return res;
#else
307 308
  (void)url;
  (void)output;
309
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
310
  return -1;
311
#endif
312 313
}

314
int CurlHelper::RetrieveFile(const std::ostringstream& urlStream, std::string filename) const
315 316 317 318 319
{
  return RetrieveFile(urlStream.str(), filename);
}

int CurlHelper::RetrieveFile(const std::string& urlString, std::string filename) const
320
{
OTB Bot's avatar
STYLE  
OTB Bot committed
321
#ifdef OTB_USE_CURL
322

323
  CURLcode res = CURLE_OK;
324

325
  CurlResource::Pointer curlResource = CurlResource::New();
OTB Bot's avatar
STYLE  
OTB Bot committed
326

327 328
  CurlFileDescriptorResource::Pointer output_file = CurlFileDescriptorResource::New();
  output_file->OpenFile(filename.c_str());
329

330
  char url[256];
331
  int len = static_cast<int>(urlString.size());
332
  strncpy( url, urlString.c_str(), len + 1 );
333

334
  otbMsgDevMacro(<< "Retrieving ( CurlHelper::RetrieveFile ): " << url );
335

336
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_URL, url));
337

338 339
  // Set timeout
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_TIMEOUT, m_Timeout));
OTB Bot's avatar
STYLE  
OTB Bot committed
340

341 342 343
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_FRESH_CONNECT, 1));
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_NOSIGNAL, 1));

344 345
  // Use our writing static function to avoid file descriptor
  // pointer crash on windows
346
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEFUNCTION,
347 348
                               &Self::CallbackWriteDataToFile));
  // Say the file where to write the received data
349
  CurlHandleError::ProcessCURLcode(curl_easy_setopt(curlResource->GetCurlResource(), CURLOPT_WRITEDATA,
350
                               (void*) output_file->GetFileResource()));
OTB Bot's avatar
STYLE  
OTB Bot committed
351

352
  CurlHandleError::ProcessCURLcode(curl_easy_perform(curlResource->GetCurlResource()));
353

354 355 356
  otbMsgDevMacro(<< " -> " << res);
  return res;
#else
357 358
  (void)urlString;
  (void)filename;
359
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
360
  return -1;
361
#endif
362 363
}

OTB Bot's avatar
STYLE  
OTB Bot committed
364
int CurlHelper::RetrieveFileMulti(const std::vector<std::string>& listURLs,
365
                                  const std::vector<std::string>& listFilename,
366
                                  int maxConnect) const
367
{
368

369
#ifdef OTB_USE_CURL
370 371
#if 0
//#ifdef OTB_CURL_MULTI_AVAILABLE
372
  otbMsgDevMacro(<< "Using curl multi");
373

374 375
  // Initialize curl handle resource
  CurlMultiResource::Pointer  multiHandle = CurlMultiResource::New();
376

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
  std::vector<CurlResource::Pointer> listCurlHandles;
  std::vector<CurlFileDescriptorResource::Pointer> listFiles;

  std::vector<std::string>::const_iterator filename;
  filename = listFilename.begin();
  while (filename != listFilename.end())
    {
    CurlFileDescriptorResource::Pointer lOutputFile = CurlFileDescriptorResource::New();
    lOutputFile->OpenFile((*filename).c_str());

    // Add file to vector
    listFiles.push_back(lOutputFile);
    ++filename;
    }

  listCurlHandles.clear();

  std::vector<std::string>::const_iterator url;
  std::vector<CurlFileDescriptorResource::Pointer>::const_iterator      file;
  url = listURLs.begin();
  file = listFiles.begin();
  while ((url != listURLs.end()) && (file != listFiles.end()))
    {
    otbMsgDevMacro(<< "Retrieving: " << (*url).data());
    CurlResource::Pointer lEasyHandle = CurlResource::New();

    // Param easy handle
404 405 406
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_USERAGENT, m_Browser.data()));
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_URL, (*url).data()));
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEFUNCTION,
407
                                 &Self::CallbackWriteDataToFile));
408
    CurlHandleError::ProcessCURLcode(curl_easy_setopt(lEasyHandle->GetCurlResource(), CURLOPT_WRITEDATA,
Otmane Lahlou's avatar
STYLE  
Otmane Lahlou committed
409
                                 (void*) (*file)->GetFileResource()));
410

411
    // Add easy handle to multi handle
412
    CurlHandleError::ProcessCURLcode(curl_multi_add_handle(multiHandle->GetCurlMultiResource(), lEasyHandle->GetCurlResource()));
413

414
    // Add handle to vector
415 416 417 418 419 420 421
    listCurlHandles.push_back(lEasyHandle);
    ++url;
    ++file;
    }

  //fetch tiles
  // Configure multi handle - set the maximum connections
422 423
  CurlHandleError::ProcessCURLcode(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_MAXCONNECTS, maxConnect));
  CurlHandleError::ProcessCURLcode(curl_multi_setopt(multiHandle->GetCurlMultiResource(), CURLMOPT_PIPELINING, 0));
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449

  // Perform
  int lStillRunning;

  while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));

  // Now get that URL
  while (lStillRunning)
    {
    struct timeval timeout;
    int            rc; // Return code

    fd_set fdread;
    fd_set fdwrite;
    fd_set fdexcep;
    int    maxfd;

    FD_ZERO(&fdread);
    FD_ZERO(&fdwrite);
    FD_ZERO(&fdexcep);

    /* set a suitable timeout to play around with */
    timeout.tv_sec = 0;
    timeout.tv_usec = 1;

    /* get file descriptors from the transfers */
450
    CurlHandleError::ProcessCURLcode(curl_multi_fdset(multiHandle->GetCurlMultiResource(), &fdread, &fdwrite, &fdexcep, &maxfd));
451 452 453 454 455 456 457 458 459 460 461 462 463 464

    rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);

    switch (rc)
      {
      case -1:
        /* select error */
        break;
      case 0:
        /* timeout */
      default:
        /* timeout or readable/writable sockets */
        while (CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multiHandle->GetCurlMultiResource(), &lStillRunning));
        break;
465
      }
466 467 468 469 470 471 472 473 474
    }

  int      remaining_msgs = 1;
  int      error = 0;
  CURLMsg *msg;
  while (remaining_msgs)
    {
    msg = curl_multi_info_read(multiHandle->GetCurlMultiResource(), &remaining_msgs);
    if (msg != NULL)
475
      {
476 477 478 479 480
      if (CURLE_OK != msg->data.result) error = 1;
      }
    }

  if (error != 0)
481
    {
482
    itkExceptionMacro(<< "otbCurlHelper: Error occurs while perform Multi handle");
483
    }
484 485 486 487

  // Cleanup
  listFiles.clear();
  listCurlHandles.clear();
Julien Malik's avatar
Julien Malik committed
488

OTB Bot's avatar
STYLE  
OTB Bot committed
489
  return 0;
490
#else
Guillaume Pasero's avatar
Guillaume Pasero committed
491
  (void)maxConnect;
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
  //fallback on non curl multi
  otbMsgDevMacro(<< "Curl multi is not available, fallback on standard");

  std::vector<std::string>::const_iterator url;
  std::vector<std::string>::const_iterator file;
  url = listURLs.begin();
  file = listFilename.begin();
  int res = 0;
  int resTmp = -1;
  while ((url != listURLs.end()) && (file != listFilename.end()))
    {
    resTmp = RetrieveFile(*url, *file);
    if (res == 0) res = resTmp;
    ++url;
    ++file;
    }

  return res;
#endif
#else
512 513 514
  (void)maxConnect;
  (void)listURLs;
  (void)listFilename;
515
  otbMsgDevMacro(<< "Curl is not available, compile with OTB_USE_CURL to ON");
516
  return -1;
517
#endif
518 519
}

520
size_t CurlHelper::CallbackWriteDataToFile(void* ptr, size_t size, size_t nmemb, void* data)
521 522 523
{
  size_t written;

OTB Bot's avatar
STYLE  
OTB Bot committed
524
  FILE * fDescriptor = (FILE *) (data);
525

OTB Bot's avatar
STYLE  
OTB Bot committed
526
  written = fwrite(ptr, size, nmemb, fDescriptor);
527 528 529 530

  return written;
}

531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
/*
size_t CurlHelper::CallbackWriteDataToCharVector(void *ptr, size_t size, size_t nmemb, void *data)
{
  register int realsize = (int)(size * nmemb);

  std::vector<char> *vec
    = static_cast<std::vector<char>*>(data);
  const char* chPtr = static_cast<char*>(ptr);
  vec->insert(vec->end(), chPtr, chPtr + realsize);

  return realsize;
}
*/

size_t CurlHelper::CallbackWriteDataToStringStream(void *ptr, size_t size, size_t nmemb, void *data)
{
  std::ostringstream& stream = *reinterpret_cast<std::ostringstream*>(data);
  stream << reinterpret_cast<char*>(ptr);
  return size * nmemb;
}

552
size_t CurlHelper::CallbackWriteDataDummy(void *itkNotUsed(ptr), size_t size, size_t nmemb, void *itkNotUsed(data))
553 554 555 556
{
  return size * nmemb;
}

557
}