downloads assets correctly

TODO: pre-1.6 versions don't work yet

1 files changed, 93 insertions, 34 deletions

diff --git a/src/downloadpool.c b/src/downloadpool.c
index 5da2905..e02e0af 100644
--- a/src/downloadpool.c
+++ b/src/downloadpool.c
@@ -1,10 +1,11 @@
 #include "downloadpool.h"
 #include "macros.h"
 #include <curl/curl.h>
-#include <curl/multi.h>
 #include <stdbool.h>
 #include <stdlib.h>
 
+#include "command.h"
+
 struct l2_dlpool__handle {
   CURL *easy;
   void *user;
@@ -15,14 +16,15 @@ struct l2_dlpool__handle {
 
 struct l2_dlpool__pool_tag {
   CURLM *multi;
+  size_t remaining_handles;
 
-  struct l2_dlpool__handle *head, *tail;
+  struct l2_dlpool__handle *head, *tail, *first_waiting;
   
   l2_dlpool_download_init_proc_t *initproc;
   l2_dlpool_one_downloaded_proc_t *dlproc;
 };
 
-l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc)
+l2_dlpool_t *l2_dlpool_init2(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc, size_t handles)
 {
   l2_dlpool_t *ret = calloc(1, sizeof(l2_dlpool_t));
   if (!ret) return NULL;
@@ -34,6 +36,7 @@ l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_
 
   ret->initproc = init;
   ret->dlproc = dlproc;
+  ret->remaining_handles = handles;
 
   return ret;
 
@@ -45,31 +48,47 @@ cleanup:
   return NULL;
 }
 
+l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc)
+{
+  return l2_dlpool_init2(init, dlproc, 50);
+}
+
+int l2_dlpool__call_init(l2_dlpool_t *pool, struct l2_dlpool__handle *handle)
+{
+  curl_easy_reset(handle->easy);
+
+  curl_easy_setopt(handle->easy, CURLOPT_USERAGENT, L2_USER_AGENT);
+
+  return (*pool->initproc)(handle->easy, handle->user);
+}
+
 int l2_dlpool_add(l2_dlpool_t *pool, void *user)
 {
   int res = -1;
   struct l2_dlpool__handle *newhandle = calloc(1, sizeof(struct l2_dlpool__handle));
   if (!newhandle) return -1;
 
-  newhandle->easy = curl_easy_init();
-  if (!newhandle->easy) goto cleanup;
+  newhandle->user = user;
 
-  curl_easy_setopt(newhandle->easy, CURLOPT_USERAGENT, L2_USER_AGENT);
+  if (pool->remaining_handles) {
+    newhandle->easy = curl_easy_init();
+    if (!newhandle->easy) goto cleanup;
 
-  if ((*pool->initproc)(newhandle->easy, user) < 0) {
-    res = 0;
-    goto cleanup;
-  }
+    --pool->remaining_handles;
 
-  newhandle->user = user;
+    if (l2_dlpool__call_init(pool, newhandle) < 0) {
+      res = 0;
+      goto cleanup;
+    }
 
-  CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy);
-  if (mc != CURLM_OK) {
-    if (mc == CURLM_OUT_OF_MEMORY) {
-      fputs("CURL out of memory. (You are doomed.)\n", stderr);
-      abort();
+    CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy);
+    if (mc != CURLM_OK) {
+      if (mc == CURLM_OUT_OF_MEMORY) {
+        fputs("CURL out of memory. (You are doomed.)\n", stderr);
+        abort();
+      }
+      goto cleanup_multi;
     }
-    goto cleanup;
   }
 
   if (pool->tail) {
@@ -80,20 +99,46 @@ int l2_dlpool_add(l2_dlpool_t *pool, void *user)
     pool->head = pool->tail = newhandle;
   }
 
+  if (!newhandle->easy && !pool->first_waiting) {
+    pool->first_waiting = newhandle;
+  }
+
   return 1;
 
+cleanup_multi:
+  curl_multi_remove_handle(pool->multi, newhandle->easy);
+
 cleanup:
   if (newhandle) {
-    if (newhandle->easy) curl_easy_cleanup(newhandle->easy);
+    if (newhandle->easy) {
+      curl_easy_cleanup(newhandle->easy);
+      ++pool->remaining_handles;
+    }
   }
   free(newhandle);
   return res;
 }
 
+void l2_dlpool__remove_handle(l2_dlpool_t *pool, struct l2_dlpool__handle *handle)
+{
+  if (handle->prev) {
+    handle->prev->next = handle->next;
+  } else {
+    pool->head = handle->next;
+  }
+
+  if (handle->next) {
+    handle->next->prev = handle->prev;
+  } else {
+    pool->tail = handle->prev;
+  }
+}
+
 int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code)
 {
   struct l2_dlpool__handle *handle = NULL;
   int uret = 0;
+  int ret = 1;
   for (struct l2_dlpool__handle *cur = pool->head; cur; cur = cur->next) {
     if (cur->easy == easy) {
       handle = cur;
@@ -104,27 +149,36 @@ int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code)
   if (handle) {
     uret = (*pool->dlproc)(easy, code, handle->user);
 
-    if (handle->prev) {
-      handle->prev->next = handle->next;
-    } else {
-      pool->head = handle->next;
-    }
-
-    if (handle->next) {
-      handle->next->prev = handle->prev;
-    } else {
-      pool->tail = handle->prev;
-    }
+    l2_dlpool__remove_handle(pool, handle);
 
     free(handle);
     handle = NULL;
   }
 
   curl_multi_remove_handle(pool->multi, easy);
-  curl_easy_cleanup(easy);
+
+  while (pool->first_waiting) {
+    handle = pool->first_waiting;
+    pool->first_waiting = pool->first_waiting->next;
+
+    handle->easy = easy;
+    if (l2_dlpool__call_init(pool, handle) < 0) {
+      l2_dlpool__remove_handle(pool, handle);
+      free(handle);
+      handle = NULL;
+      ret = 0;
+    } else {
+      curl_multi_add_handle(pool->multi, easy);
+      break;
+    }
+  }
+
+  if (!handle) { /* we have not successfully repurposed this handle */
+    curl_easy_cleanup(easy);
+  }
 
   if (uret < 0) return -1;
-  return 0;
+  return ret;
 }
 
 int l2_dlpool_perform(l2_dlpool_t *pool)
@@ -133,6 +187,7 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
   int running = 0;
   int queuelen = 0;
   CURLMsg *msg;
+  int res = 1, ures;
 
   do {
     code = curl_multi_perform(pool->multi, &running);
@@ -142,7 +197,11 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
 
     while ((msg = curl_multi_info_read(pool->multi, &queuelen))) {
       if (msg->msg == CURLMSG_DONE) {
-        if (l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result) < 0) return -1;
+        if ((ures = l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result)) < 0) {
+          return -1;
+        } else if (ures == 0) {
+          res = 0;
+        }
       }
     }
 
@@ -150,9 +209,9 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
     if (code != CURLM_OK) {
       return -1;
     }
-  } while (running);
+  } while (pool->head);
 
-  return 0;
+  return res;
 }
 
 void l2_dlpool_cleanup(l2_dlpool_t *pool)