diff options
Diffstat (limited to 'src/downloadpool.c')
| -rw-r--r-- | src/downloadpool.c | 127 |
1 files changed, 93 insertions, 34 deletions
diff --git a/src/downloadpool.c b/src/downloadpool.c index 5da2905..e02e0af 100644 --- a/src/downloadpool.c +++ b/src/downloadpool.c @@ -1,10 +1,11 @@ #include "downloadpool.h" #include "macros.h" #include <curl/curl.h> -#include <curl/multi.h> #include <stdbool.h> #include <stdlib.h> +#include "command.h" + struct l2_dlpool__handle { CURL *easy; void *user; @@ -15,14 +16,15 @@ struct l2_dlpool__handle { struct l2_dlpool__pool_tag { CURLM *multi; + size_t remaining_handles; - struct l2_dlpool__handle *head, *tail; + struct l2_dlpool__handle *head, *tail, *first_waiting; l2_dlpool_download_init_proc_t *initproc; l2_dlpool_one_downloaded_proc_t *dlproc; }; -l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc) +l2_dlpool_t *l2_dlpool_init2(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc, size_t handles) { l2_dlpool_t *ret = calloc(1, sizeof(l2_dlpool_t)); if (!ret) return NULL; @@ -34,6 +36,7 @@ l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_ ret->initproc = init; ret->dlproc = dlproc; + ret->remaining_handles = handles; return ret; @@ -45,31 +48,47 @@ cleanup: return NULL; } +l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc) +{ + return l2_dlpool_init2(init, dlproc, 50); +} + +int l2_dlpool__call_init(l2_dlpool_t *pool, struct l2_dlpool__handle *handle) +{ + curl_easy_reset(handle->easy); + + curl_easy_setopt(handle->easy, CURLOPT_USERAGENT, L2_USER_AGENT); + + return (*pool->initproc)(handle->easy, handle->user); +} + int l2_dlpool_add(l2_dlpool_t *pool, void *user) { int res = -1; struct l2_dlpool__handle *newhandle = calloc(1, sizeof(struct l2_dlpool__handle)); if (!newhandle) return -1; - newhandle->easy = curl_easy_init(); - if (!newhandle->easy) goto cleanup; + newhandle->user = user; - curl_easy_setopt(newhandle->easy, CURLOPT_USERAGENT, L2_USER_AGENT); + if (pool->remaining_handles) { + newhandle->easy = curl_easy_init(); + if (!newhandle->easy) goto cleanup; - if ((*pool->initproc)(newhandle->easy, user) < 0) { - res = 0; - goto cleanup; - } + --pool->remaining_handles; - newhandle->user = user; + if (l2_dlpool__call_init(pool, newhandle) < 0) { + res = 0; + goto cleanup; + } - CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy); - if (mc != CURLM_OK) { - if (mc == CURLM_OUT_OF_MEMORY) { - fputs("CURL out of memory. (You are doomed.)\n", stderr); - abort(); + CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy); + if (mc != CURLM_OK) { + if (mc == CURLM_OUT_OF_MEMORY) { + fputs("CURL out of memory. (You are doomed.)\n", stderr); + abort(); + } + goto cleanup_multi; } - goto cleanup; } if (pool->tail) { @@ -80,20 +99,46 @@ int l2_dlpool_add(l2_dlpool_t *pool, void *user) pool->head = pool->tail = newhandle; } + if (!newhandle->easy && !pool->first_waiting) { + pool->first_waiting = newhandle; + } + return 1; +cleanup_multi: + curl_multi_remove_handle(pool->multi, newhandle->easy); + cleanup: if (newhandle) { - if (newhandle->easy) curl_easy_cleanup(newhandle->easy); + if (newhandle->easy) { + curl_easy_cleanup(newhandle->easy); + ++pool->remaining_handles; + } } free(newhandle); return res; } +void l2_dlpool__remove_handle(l2_dlpool_t *pool, struct l2_dlpool__handle *handle) +{ + if (handle->prev) { + handle->prev->next = handle->next; + } else { + pool->head = handle->next; + } + + if (handle->next) { + handle->next->prev = handle->prev; + } else { + pool->tail = handle->prev; + } +} + int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code) { struct l2_dlpool__handle *handle = NULL; int uret = 0; + int ret = 1; for (struct l2_dlpool__handle *cur = pool->head; cur; cur = cur->next) { if (cur->easy == easy) { handle = cur; @@ -104,27 +149,36 @@ int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code) if (handle) { uret = (*pool->dlproc)(easy, code, handle->user); - if (handle->prev) { - handle->prev->next = handle->next; - } else { - pool->head = handle->next; - } - - if (handle->next) { - handle->next->prev = handle->prev; - } else { - pool->tail = handle->prev; - } + l2_dlpool__remove_handle(pool, handle); free(handle); handle = NULL; } curl_multi_remove_handle(pool->multi, easy); - curl_easy_cleanup(easy); + + while (pool->first_waiting) { + handle = pool->first_waiting; + pool->first_waiting = pool->first_waiting->next; + + handle->easy = easy; + if (l2_dlpool__call_init(pool, handle) < 0) { + l2_dlpool__remove_handle(pool, handle); + free(handle); + handle = NULL; + ret = 0; + } else { + curl_multi_add_handle(pool->multi, easy); + break; + } + } + + if (!handle) { /* we have not successfully repurposed this handle */ + curl_easy_cleanup(easy); + } if (uret < 0) return -1; - return 0; + return ret; } int l2_dlpool_perform(l2_dlpool_t *pool) @@ -133,6 +187,7 @@ int l2_dlpool_perform(l2_dlpool_t *pool) int running = 0; int queuelen = 0; CURLMsg *msg; + int res = 1, ures; do { code = curl_multi_perform(pool->multi, &running); @@ -142,7 +197,11 @@ int l2_dlpool_perform(l2_dlpool_t *pool) while ((msg = curl_multi_info_read(pool->multi, &queuelen))) { if (msg->msg == CURLMSG_DONE) { - if (l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result) < 0) return -1; + if ((ures = l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result)) < 0) { + return -1; + } else if (ures == 0) { + res = 0; + } } } @@ -150,9 +209,9 @@ int l2_dlpool_perform(l2_dlpool_t *pool) if (code != CURLM_OK) { return -1; } - } while (running); + } while (pool->head); - return 0; + return res; } void l2_dlpool_cleanup(l2_dlpool_t *pool) |