block/curl.c

   1 /*
   2  * QEMU Block driver for CURL images
   3  *
   4  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu-common.h"
  25 #include "block_int.h"
  26 #include <curl/curl.h>
  27
  28 // #define DEBUG
  29 // #define DEBUG_VERBOSE
  30
  31 #ifdef DEBUG_CURL
  32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
  33 #else
  34 #define DPRINTF(fmt, ...) do { } while (0)
  35 #endif
  36
  37 #define CURL_NUM_STATES 8
  38 #define CURL_NUM_ACB    8
  39 #define SECTOR_SIZE     512
  40 #define READ_AHEAD_SIZE (256 * 1024)
  41
  42 #define FIND_RET_NONE   0
  43 #define FIND_RET_OK     1
  44 #define FIND_RET_WAIT   2
  45
  46 struct BDRVCURLState;
  47
  48 typedef struct CURLAIOCB {
  49     BlockDriverAIOCB common;
  50     QEMUIOVector *qiov;
  51     size_t start;
  52     size_t end;
  53 } CURLAIOCB;
  54
  55 typedef struct CURLState
  56 {
  57     struct BDRVCURLState *s;
  58     CURLAIOCB *acb[CURL_NUM_ACB];
  59     CURL *curl;
  60     char *orig_buf;
  61     size_t buf_start;
  62     size_t buf_off;
  63     size_t buf_len;
  64     char range[128];
  65     char errmsg[CURL_ERROR_SIZE];
  66     char in_use;
  67 } CURLState;
  68
  69 typedef struct BDRVCURLState {
  70     CURLM *multi;
  71     size_t len;
  72     CURLState states[CURL_NUM_STATES];
  73     char *url;
  74     size_t readahead_size;
  75 } BDRVCURLState;
  76
  77 static void curl_clean_state(CURLState *s);
  78 static void curl_multi_do(void *arg);
  79 static int curl_aio_flush(void *opaque);
  80
  81 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
  82                         void *s, void *sp)
  83 {
  84     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
  85     switch (action) {
  86         case CURL_POLL_IN:
  87             qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, curl_aio_flush,
  88                                     NULL, s);
  89             break;
  90         case CURL_POLL_OUT:
  91             qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, curl_aio_flush,
  92                                     NULL, s);
  93             break;
  94         case CURL_POLL_INOUT:
  95             qemu_aio_set_fd_handler(fd, curl_multi_do, curl_multi_do,
  96                                     curl_aio_flush, NULL, s);
  97             break;
  98         case CURL_POLL_REMOVE:
  99             qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL, NULL);
 100             break;
 101     }
 102
 103     return 0;
 104 }
 105
 106 static size_t curl_size_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 107 {
 108     CURLState *s = ((CURLState*)opaque);
 109     size_t realsize = size * nmemb;
 110     size_t fsize;
 111
 112     if(sscanf(ptr, "Content-Length: %zd", &fsize) == 1) {
 113         s->s->len = fsize;
 114     }
 115
 116     return realsize;
 117 }
 118
 119 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 120 {
 121     CURLState *s = ((CURLState*)opaque);
 122     size_t realsize = size * nmemb;
 123     int i;
 124
 125     DPRINTF("CURL: Just reading %zd bytes\n", realsize);
 126
 127     if (!s || !s->orig_buf)
 128         goto read_end;
 129
 130     memcpy(s->orig_buf + s->buf_off, ptr, realsize);
 131     s->buf_off += realsize;
 132
 133     for(i=0; i<CURL_NUM_ACB; i++) {
 134         CURLAIOCB *acb = s->acb[i];
 135
 136         if (!acb)
 137             continue;
 138
 139         if ((s->buf_off >= acb->end)) {
 140             qemu_iovec_from_buffer(acb->qiov, s->orig_buf + acb->start,
 141                                    acb->end - acb->start);
 142             acb->common.cb(acb->common.opaque, 0);
 143             qemu_aio_release(acb);
 144             s->acb[i] = NULL;
 145         }
 146     }
 147
 148 read_end:
 149     return realsize;
 150 }
 151
 152 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
 153                          CURLAIOCB *acb)
 154 {
 155     int i;
 156     size_t end = start + len;
 157
 158     for (i=0; i<CURL_NUM_STATES; i++) {
 159         CURLState *state = &s->states[i];
 160         size_t buf_end = (state->buf_start + state->buf_off);
 161         size_t buf_fend = (state->buf_start + state->buf_len);
 162
 163         if (!state->orig_buf)
 164             continue;
 165         if (!state->buf_off)
 166             continue;
 167
 168         // Does the existing buffer cover our section?
 169         if ((start >= state->buf_start) &&
 170             (start <= buf_end) &&
 171             (end >= state->buf_start) &&
 172             (end <= buf_end))
 173         {
 174             char *buf = state->orig_buf + (start - state->buf_start);
 175
 176             qemu_iovec_from_buffer(acb->qiov, buf, len);
 177             acb->common.cb(acb->common.opaque, 0);
 178
 179             return FIND_RET_OK;
 180         }
 181
 182         // Wait for unfinished chunks
 183         if ((start >= state->buf_start) &&
 184             (start <= buf_fend) &&
 185             (end >= state->buf_start) &&
 186             (end <= buf_fend))
 187         {
 188             int j;
 189
 190             acb->start = start - state->buf_start;
 191             acb->end = acb->start + len;
 192
 193             for (j=0; j<CURL_NUM_ACB; j++) {
 194                 if (!state->acb[j]) {
 195                     state->acb[j] = acb;
 196                     return FIND_RET_WAIT;
 197                 }
 198             }
 199         }
 200     }
 201
 202     return FIND_RET_NONE;
 203 }
 204
 205 static void curl_multi_do(void *arg)
 206 {
 207     BDRVCURLState *s = (BDRVCURLState *)arg;
 208     int running;
 209     int r;
 210     int msgs_in_queue;
 211
 212     if (!s->multi)
 213         return;
 214
 215     do {
 216         r = curl_multi_socket_all(s->multi, &running);
 217     } while(r == CURLM_CALL_MULTI_PERFORM);
 218
 219     /* Try to find done transfers, so we can free the easy
 220      * handle again. */
 221     do {
 222         CURLMsg *msg;
 223         msg = curl_multi_info_read(s->multi, &msgs_in_queue);
 224
 225         if (!msg)
 226             break;
 227         if (msg->msg == CURLMSG_NONE)
 228             break;
 229
 230         switch (msg->msg) {
 231             case CURLMSG_DONE:
 232             {
 233                 CURLState *state = NULL;
 234                 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&state);
 235
 236                 /* ACBs for successful messages get completed in curl_read_cb */
 237                 if (msg->data.result != CURLE_OK) {
 238                     int i;
 239                     for (i = 0; i < CURL_NUM_ACB; i++) {
 240                         CURLAIOCB *acb = state->acb[i];
 241
 242                         if (acb == NULL) {
 243                             continue;
 244                         }
 245
 246                         acb->common.cb(acb->common.opaque, -EIO);
 247                         qemu_aio_release(acb);
 248                         state->acb[i] = NULL;
 249                     }
 250                 }
 251
 252                 curl_clean_state(state);
 253                 break;
 254             }
 255             default:
 256                 msgs_in_queue = 0;
 257                 break;
 258         }
 259     } while(msgs_in_queue);
 260 }
 261
 262 static CURLState *curl_init_state(BDRVCURLState *s)
 263 {
 264     CURLState *state = NULL;
 265     int i, j;
 266
 267     do {
 268         for (i=0; i<CURL_NUM_STATES; i++) {
 269             for (j=0; j<CURL_NUM_ACB; j++)
 270                 if (s->states[i].acb[j])
 271                     continue;
 272             if (s->states[i].in_use)
 273                 continue;
 274
 275             state = &s->states[i];
 276             state->in_use = 1;
 277             break;
 278         }
 279         if (!state) {
 280             usleep(100);
 281             curl_multi_do(s);
 282         }
 283     } while(!state);
 284
 285     if (state->curl)
 286         goto has_curl;
 287
 288     state->curl = curl_easy_init();
 289     if (!state->curl)
 290         return NULL;
 291     curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
 292     curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
 293     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 294     curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
 295     curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
 296     curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
 297     curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
 298     curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
 299     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
 300     curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 301
 302 #ifdef DEBUG_VERBOSE
 303     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
 304 #endif
 305
 306 has_curl:
 307
 308     state->s = s;
 309
 310     return state;
 311 }
 312
 313 static void curl_clean_state(CURLState *s)
 314 {
 315     if (s->s->multi)
 316         curl_multi_remove_handle(s->s->multi, s->curl);
 317     s->in_use = 0;
 318 }
 319
 320 static int curl_open(BlockDriverState *bs, const char *filename, int flags)
 321 {
 322     BDRVCURLState *s = bs->opaque;
 323     CURLState *state = NULL;
 324     double d;
 325
 326     #define RA_OPTSTR ":readahead="
 327     char *file;
 328     char *ra;
 329     const char *ra_val;
 330     int parse_state = 0;
 331
 332     static int inited = 0;
 333
 334     file = g_strdup(filename);
 335     s->readahead_size = READ_AHEAD_SIZE;
 336
 337     /* Parse a trailing ":readahead=#:" param, if present. */
 338     ra = file + strlen(file) - 1;
 339     while (ra >= file) {
 340         if (parse_state == 0) {
 341             if (*ra == ':')
 342                 parse_state++;
 343             else
 344                 break;
 345         } else if (parse_state == 1) {
 346             if (*ra > '9' || *ra < '0') {
 347                 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
 348                 if (opt_start > file &&
 349                     strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
 350                     ra_val = ra + 1;
 351                     ra -= strlen(RA_OPTSTR) - 1;
 352                     *ra = '\0';
 353                     s->readahead_size = atoi(ra_val);
 354                     break;
 355                 } else {
 356                     break;
 357                 }
 358             }
 359         }
 360         ra--;
 361     }
 362
 363     if ((s->readahead_size & 0x1ff) != 0) {
 364         fprintf(stderr, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512\n",
 365                 s->readahead_size);
 366         goto out_noclean;
 367     }
 368
 369     if (!inited) {
 370         curl_global_init(CURL_GLOBAL_ALL);
 371         inited = 1;
 372     }
 373
 374     DPRINTF("CURL: Opening %s\n", file);
 375     s->url = file;
 376     state = curl_init_state(s);
 377     if (!state)
 378         goto out_noclean;
 379
 380     // Get file size
 381
 382     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
 383     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_size_cb);
 384     if (curl_easy_perform(state->curl))
 385         goto out;
 386     curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 387     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 388     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
 389     if (d)
 390         s->len = (size_t)d;
 391     else if(!s->len)
 392         goto out;
 393     DPRINTF("CURL: Size = %zd\n", s->len);
 394
 395     curl_clean_state(state);
 396     curl_easy_cleanup(state->curl);
 397     state->curl = NULL;
 398
 399     // Now we know the file exists and its size, so let's
 400     // initialize the multi interface!
 401
 402     s->multi = curl_multi_init();
 403     curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
 404     curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb );
 405     curl_multi_do(s);
 406
 407     return 0;
 408
 409 out:
 410     fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
 411     curl_easy_cleanup(state->curl);
 412     state->curl = NULL;
 413 out_noclean:
 414     g_free(file);
 415     return -EINVAL;
 416 }
 417
 418 static int curl_aio_flush(void *opaque)
 419 {
 420     BDRVCURLState *s = opaque;
 421     int i, j;
 422
 423     for (i=0; i < CURL_NUM_STATES; i++) {
 424         for(j=0; j < CURL_NUM_ACB; j++) {
 425             if (s->states[i].acb[j]) {
 426                 return 1;
 427             }
 428         }
 429     }
 430     return 0;
 431 }
 432
 433 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
 434 {
 435     // Do we have to implement canceling? Seems to work without...
 436 }
 437
 438 static AIOPool curl_aio_pool = {
 439     .aiocb_size         = sizeof(CURLAIOCB),
 440     .cancel             = curl_aio_cancel,
 441 };
 442
 443 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 444         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
 445         BlockDriverCompletionFunc *cb, void *opaque)
 446 {
 447     BDRVCURLState *s = bs->opaque;
 448     CURLAIOCB *acb;
 449     size_t start = sector_num * SECTOR_SIZE;
 450     size_t end;
 451     CURLState *state;
 452
 453     acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
 454     if (!acb)
 455         return NULL;
 456
 457     acb->qiov = qiov;
 458
 459     // In case we have the requested data already (e.g. read-ahead),
 460     // we can just call the callback and be done.
 461
 462     switch (curl_find_buf(s, start, nb_sectors * SECTOR_SIZE, acb)) {
 463         case FIND_RET_OK:
 464             qemu_aio_release(acb);
 465             // fall through
 466         case FIND_RET_WAIT:
 467             return &acb->common;
 468         default:
 469             break;
 470     }
 471
 472     // No cache found, so let's start a new request
 473
 474     state = curl_init_state(s);
 475     if (!state)
 476         return NULL;
 477
 478     acb->start = 0;
 479     acb->end = (nb_sectors * SECTOR_SIZE);
 480
 481     state->buf_off = 0;
 482     if (state->orig_buf)
 483         g_free(state->orig_buf);
 484     state->buf_start = start;
 485     state->buf_len = acb->end + s->readahead_size;
 486     end = MIN(start + state->buf_len, s->len) - 1;
 487     state->orig_buf = g_malloc(state->buf_len);
 488     state->acb[0] = acb;
 489
 490     snprintf(state->range, 127, "%zd-%zd", start, end);
 491     DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
 492             (nb_sectors * SECTOR_SIZE), start, state->range);
 493     curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
 494
 495     curl_multi_add_handle(s->multi, state->curl);
 496     curl_multi_do(s);
 497
 498     return &acb->common;
 499 }
 500
 501 static void curl_close(BlockDriverState *bs)
 502 {
 503     BDRVCURLState *s = bs->opaque;
 504     int i;
 505
 506     DPRINTF("CURL: Close\n");
 507     for (i=0; i<CURL_NUM_STATES; i++) {
 508         if (s->states[i].in_use)
 509             curl_clean_state(&s->states[i]);
 510         if (s->states[i].curl) {
 511             curl_easy_cleanup(s->states[i].curl);
 512             s->states[i].curl = NULL;
 513         }
 514         if (s->states[i].orig_buf) {
 515             g_free(s->states[i].orig_buf);
 516             s->states[i].orig_buf = NULL;
 517         }
 518     }
 519     if (s->multi)
 520         curl_multi_cleanup(s->multi);
 521     if (s->url)
 522         free(s->url);
 523 }
 524
 525 static int64_t curl_getlength(BlockDriverState *bs)
 526 {
 527     BDRVCURLState *s = bs->opaque;
 528     return s->len;
 529 }
 530
 531 static BlockDriver bdrv_http = {
 532     .format_name     = "http",
 533     .protocol_name   = "http",
 534
 535     .instance_size   = sizeof(BDRVCURLState),
 536     .bdrv_file_open  = curl_open,
 537     .bdrv_close      = curl_close,
 538     .bdrv_getlength  = curl_getlength,
 539
 540     .bdrv_aio_readv  = curl_aio_readv,
 541 };
 542
 543 static BlockDriver bdrv_https = {
 544     .format_name     = "https",
 545     .protocol_name   = "https",
 546
 547     .instance_size   = sizeof(BDRVCURLState),
 548     .bdrv_file_open  = curl_open,
 549     .bdrv_close      = curl_close,
 550     .bdrv_getlength  = curl_getlength,
 551
 552     .bdrv_aio_readv  = curl_aio_readv,
 553 };
 554
 555 static BlockDriver bdrv_ftp = {
 556     .format_name     = "ftp",
 557     .protocol_name   = "ftp",
 558
 559     .instance_size   = sizeof(BDRVCURLState),
 560     .bdrv_file_open  = curl_open,
 561     .bdrv_close      = curl_close,
 562     .bdrv_getlength  = curl_getlength,
 563
 564     .bdrv_aio_readv  = curl_aio_readv,
 565 };
 566
 567 static BlockDriver bdrv_ftps = {
 568     .format_name     = "ftps",
 569     .protocol_name   = "ftps",
 570
 571     .instance_size   = sizeof(BDRVCURLState),
 572     .bdrv_file_open  = curl_open,
 573     .bdrv_close      = curl_close,
 574     .bdrv_getlength  = curl_getlength,
 575
 576     .bdrv_aio_readv  = curl_aio_readv,
 577 };
 578
 579 static BlockDriver bdrv_tftp = {
 580     .format_name     = "tftp",
 581     .protocol_name   = "tftp",
 582
 583     .instance_size   = sizeof(BDRVCURLState),
 584     .bdrv_file_open  = curl_open,
 585     .bdrv_close      = curl_close,
 586     .bdrv_getlength  = curl_getlength,
 587
 588     .bdrv_aio_readv  = curl_aio_readv,
 589 };
 590
 591 static void curl_block_init(void)
 592 {
 593     bdrv_register(&bdrv_http);
 594     bdrv_register(&bdrv_https);
 595     bdrv_register(&bdrv_ftp);
 596     bdrv_register(&bdrv_ftps);
 597     bdrv_register(&bdrv_tftp);
 598 }
 599
 600 block_init(curl_block_init);