downloads assets correctly

TODO: pre-1.6 versions don't work yet

6 files changed, 378 insertions, 37 deletions

diff --git a/src/assets.c b/src/assets.c
index 57624ae..7699d33 100644
--- a/src/assets.c
+++ b/src/assets.c
@@ -1,8 +1,281 @@
 #include "assets.h"
+#include "digest/digest.h"
 #include "version.h"
 #include "l2su.h"
+#include "downloadpool.h"
+#include "macros.h"
+#include "command.h"
 
+#include <stdio.h>
+#include <string.h>
+#include <stdio.h>
 #include <jansson.h>
+#include <curl/curl.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <unistd.h>
+
+struct l2_assets__download_info {
+  char *url;
+  const char *name;
+  char *fullpath;
+
+  size_t expect_size;
+  l2_sha1_digest_t expect_digest;
+
+  /* initialized in init function */
+  FILE *fp;
+  l2_sha1_state_t digest_state;
+  size_t recv_size;
+
+  char errbuf[CURL_ERROR_SIZE];
+  
+  struct l2_assets__download_info *next;
+};
+
+size_t l2_assets__download_asset_dlcb(char *buf, size_t size, size_t nmemb, void *user)
+{
+  struct l2_assets__download_info *info = user;
+  size_t realsz = size * nmemb;
+
+  if (fwrite(buf, size, nmemb, info->fp) < nmemb) {
+    return CURL_WRITEFUNC_ERROR;
+  }
+
+  l2_sha1_update(&info->digest_state, buf, realsz);
+  info->recv_size += realsz;
+  return realsz;
+}
+
+int l2_assets__download_asset_init(CURL *curl, void *user)
+{
+  struct l2_assets__download_info *info = user;
+
+  l2_sha1_init(&info->digest_state);
+
+  if (l2_launcher_mkdir_parents_ex(info->fullpath, 1) < 0) {
+    CMD_WARN("Error creating parent directories for %s (%s): %s", info->fullpath, info->name, strerror(errno));
+    return -1;
+  }
+
+  info->fp = fopen(info->fullpath, "wb");
+  if (!info->fp) {
+    CMD_WARN("Error opening %s (%s) for writing: %s", info->fullpath, info->name, strerror(errno));
+    return -1;
+  }
+
+  CMD_DEBUG("asset url %s", info->url);
+
+  curl_easy_setopt(curl, CURLOPT_URL, info->url);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, info);
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &l2_assets__download_asset_dlcb);
+  curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, info->errbuf);
+
+  return 0;
+}
+
+int l2_assets__download_asset_complete(CURL *curl, CURLcode code, void *user)
+{
+  struct l2_assets__download_info *info = user;
+  l2_sha1_digest_t dg;
+
+  fclose(info->fp);
+  info->fp = NULL;
+
+  if (code != CURLE_OK) {
+    CMD_WARN("Failed to download asset %s: %s", info->name, curl_easy_strerror(code));
+    CMD_DEBUG("Info for %s: %s", info->name, info->errbuf);
+    return -1;
+  }
+
+  long rescode;
+  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &rescode);
+  if (rescode / 100 != 2) {
+    CMD_WARN("Failed to download asset %s: server responded with %ld", info->name, rescode);
+    return -1;
+  }
+
+  l2_sha1_finalize(&info->digest_state, &dg);
+  if (l2_sha1_digest_compare(&dg, &info->expect_digest)) {
+    char expstr[L2_SHA1_HEX_STRLEN + 1], gotstr[L2_SHA1_HEX_STRLEN + 1];
+    l2_sha1_digest_to_hex(&info->expect_digest, expstr);
+    l2_sha1_digest_to_hex(&dg, gotstr);
+
+    CMD_WARN("Resource %s SHA1 mismatch! (expected %s, got %s)", info->name, expstr, gotstr);
+    if (unlink(info->fullpath) < 0) {
+      CMD_WARN("Failed to delete resource %s: %s", info->name, strerror(errno));
+    }
+    return -1;
+  }
+
+  if (info->recv_size != info->expect_size) {
+    CMD_WARN("Resource %s has wrong size! (expected %zu bytes, got %zu bytes)", info->name, info->expect_size, info->recv_size);
+    if (unlink(info->fullpath) < 0) {
+      CMD_WARN("Failed to delete resource %s: %s", info->name, strerror(errno));
+    }
+    return -1;
+  }
+
+  return 0;
+}
+
+void l2_assets__free_download_info(struct l2_assets__download_info *info)
+{
+  if (info->fp) {
+    fclose(info->fp);
+    if (unlink(info->fullpath) < 0) {
+      CMD_WARN("Failed to delete partially downloaded asset %s (%s): %s", info->name, info->fullpath, strerror(errno));
+    }
+  }
+
+  free(info->url);
+  free(info->fullpath);
+  free(info);
+}
+
+int l2_assets__download_assets(json_t *asset_index, char **path)
+{
+  int res = -1;
+  json_t *objects;
+  int virtual = 0;
+  int map_to_resources = 0;
+  l2_dlpool_t *pool = NULL;
+
+  char *resbasepath = NULL;
+
+  const char *key;
+  json_t *asset;
+  struct l2_assets__download_info *dl_info_head, *dl_info_tail;
+
+  if (json_unpack(asset_index, "{s:o, s?:b, s?:b}", "objects", &objects, "virtual", &virtual, "map_to_resources", &map_to_resources) < 0) {
+    CMD_WARN0("Malformed asset index!");
+    return -1;
+  }
+
+  virtual |= map_to_resources; /* they're the same for our purposes */
+
+  dl_info_head = dl_info_tail = NULL;
+
+  pool = l2_dlpool_init(&l2_assets__download_asset_init, &l2_assets__download_asset_complete);
+  if (!pool) {
+    res = -1;
+    goto cleanup;
+  }
+
+  json_object_foreach(objects, key, asset) {
+    /* key is path */
+
+    const char *digeststr;
+    char urldgpart[L2_SHA1_HEX_STRLEN + 1];
+    json_int_t size;
+    l2_sha1_digest_t digest;
+
+    if (json_unpack(asset, "{s:s, s:I}", "hash", &digeststr, "size", &size) < 0) {
+      CMD_WARN("Malformed asset object %s!", key);
+      res = -1;
+      goto cleanup;
+    }
+
+    if (size <= 0) {
+      CMD_WARN("Asset object %s has nonpositive size (%" JSON_INTEGER_FORMAT ")!", key, size);
+      res = -1;
+      goto cleanup;
+    }
+
+    if ((res = l2_sha1_digest_from_hex(&digest, digeststr)) < 0) {
+      CMD_WARN("Asset object %s has invalid SHA1!", key);
+      res = -1;
+      goto cleanup;
+    }
+
+    l2_sha1_digest_to_hex(&digest, urldgpart);
+    char *path = l2_launcher_sprintf_alloc("%s/assets/objects/%2.2s/%s", l2_state.paths.data, urldgpart, urldgpart);
+    int shoulddl = l2_launcher_should_download(path, &digest, size);
+    if (shoulddl < 0) {
+      free(path);
+      res = -1;
+      goto cleanup;
+    } else if (!shoulddl) {
+      CMD_DEBUG("Not downloading asset %s", key);
+      free(path);
+      continue;
+    }
+
+    struct l2_assets__download_info *curinfo = calloc(1, sizeof(struct l2_assets__download_info));
+    if (!curinfo) {
+      free(path);
+      res = -1;
+      goto cleanup;
+    }
+
+    if (dl_info_tail) {
+      dl_info_tail->next = curinfo;
+      dl_info_tail = curinfo;
+    } else {
+      dl_info_head = dl_info_tail = curinfo;
+    }
+
+    l2_sha1_digest_copy(&curinfo->expect_digest, &digest);
+    curinfo->expect_size = (size_t)size;
+
+    curinfo->url = l2_launcher_sprintf_alloc("%s/%2.2s/%s", L2_URL_RESOURCES_BASE, urldgpart, urldgpart);
+    curinfo->fullpath = path;
+    if (!curinfo->url) {
+      res = -1;
+      goto cleanup;
+    }
+
+    curinfo->name = key;
+
+    if ((res = l2_dlpool_add(pool, curinfo)) <= 0) {
+      res = -1;
+      goto cleanup;
+    }
+  }
+
+  if (dl_info_head) {
+    if (l2_dlpool_perform(pool) <= 0) {
+      res = -1;
+      goto cleanup;
+    }
+  } else {
+    CMD_INFO0("No assets to download.");
+    res = 0;
+  }
+
+  if (virtual) {
+    resbasepath = l2_launcher_sprintf_alloc("%s/assets/virtual", l2_state.paths.data);
+
+    CMD_ERROR0("virtual resources are not yet implemented. please nag figboot");
+    res = -1;
+    goto cleanup;
+  } else {
+    resbasepath = l2_launcher_sprintf_alloc("%s/assets/objects", l2_state.paths.data);
+  }
+
+  if (!resbasepath) {
+    res = -1;
+    goto cleanup;
+  }
+
+  *path = resbasepath;
+  resbasepath = NULL; /* prevent out variable from being freed */
+
+  CMD_INFO0("Successfully downloaded assets.");
+  res = 0;
+
+cleanup:
+  if (resbasepath) free(resbasepath);
+  if (pool) l2_dlpool_cleanup(pool);
+  if (dl_info_head) {
+    for (struct l2_assets__download_info *cur = dl_info_head, *temp; cur; cur = temp) {
+      temp = cur->next;
+      l2_assets__free_download_info(cur);
+    }
+  }
+
+  return res;
+}
 
 int l2_assets__download_index(json_t *version, char **path)
 {
diff --git a/src/cmd-version.c b/src/cmd-version.c
index b11e35c..bb02d90 100644
--- a/src/cmd-version.c
+++ b/src/cmd-version.c
@@ -42,6 +42,7 @@ bool feat_match_cb(const char *name, json_t *js) {
 }
 
 int l2_assets__load_index(json_t *version, json_t **asset_index);
+int l2_assets__download_assets(json_t *asset_index, char **path);
 
 unsigned cmd_version_install(struct l2_context_node *ctx, char **args)
 {
@@ -80,9 +81,15 @@ unsigned cmd_version_install(struct l2_context_node *ctx, char **args)
     CMD_FATAL0("Failed to load asset index.");
   }
 
-  json_dumpf(assets, stdout, JSON_INDENT(4));
-  putchar('\n');
+  CMD_INFO0("Downloading assets...");
+  char *assetsbase = NULL;
+  if (l2_assets__download_assets(assets, &assetsbase) < 0) {
+    CMD_FATAL0("Failed to download assets.");
+  }
+
+  CMD_INFO("Assets base: %s", assetsbase);
 
+  free(assetsbase);
   json_decref(assets);
 
   l2_version_free_libraries(libs);
diff --git a/src/downloadpool.c b/src/downloadpool.c
index 5da2905..e02e0af 100644
--- a/src/downloadpool.c
+++ b/src/downloadpool.c
@@ -1,10 +1,11 @@
 #include "downloadpool.h"
 #include "macros.h"
 #include <curl/curl.h>
-#include <curl/multi.h>
 #include <stdbool.h>
 #include <stdlib.h>
 
+#include "command.h"
+
 struct l2_dlpool__handle {
   CURL *easy;
   void *user;
@@ -15,14 +16,15 @@ struct l2_dlpool__handle {
 
 struct l2_dlpool__pool_tag {
   CURLM *multi;
+  size_t remaining_handles;
 
-  struct l2_dlpool__handle *head, *tail;
+  struct l2_dlpool__handle *head, *tail, *first_waiting;
   
   l2_dlpool_download_init_proc_t *initproc;
   l2_dlpool_one_downloaded_proc_t *dlproc;
 };
 
-l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc)
+l2_dlpool_t *l2_dlpool_init2(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc, size_t handles)
 {
   l2_dlpool_t *ret = calloc(1, sizeof(l2_dlpool_t));
   if (!ret) return NULL;
@@ -34,6 +36,7 @@ l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_
 
   ret->initproc = init;
   ret->dlproc = dlproc;
+  ret->remaining_handles = handles;
 
   return ret;
 
@@ -45,31 +48,47 @@ cleanup:
   return NULL;
 }
 
+l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc)
+{
+  return l2_dlpool_init2(init, dlproc, 50);
+}
+
+int l2_dlpool__call_init(l2_dlpool_t *pool, struct l2_dlpool__handle *handle)
+{
+  curl_easy_reset(handle->easy);
+
+  curl_easy_setopt(handle->easy, CURLOPT_USERAGENT, L2_USER_AGENT);
+
+  return (*pool->initproc)(handle->easy, handle->user);
+}
+
 int l2_dlpool_add(l2_dlpool_t *pool, void *user)
 {
   int res = -1;
   struct l2_dlpool__handle *newhandle = calloc(1, sizeof(struct l2_dlpool__handle));
   if (!newhandle) return -1;
 
-  newhandle->easy = curl_easy_init();
-  if (!newhandle->easy) goto cleanup;
+  newhandle->user = user;
 
-  curl_easy_setopt(newhandle->easy, CURLOPT_USERAGENT, L2_USER_AGENT);
+  if (pool->remaining_handles) {
+    newhandle->easy = curl_easy_init();
+    if (!newhandle->easy) goto cleanup;
 
-  if ((*pool->initproc)(newhandle->easy, user) < 0) {
-    res = 0;
-    goto cleanup;
-  }
+    --pool->remaining_handles;
 
-  newhandle->user = user;
+    if (l2_dlpool__call_init(pool, newhandle) < 0) {
+      res = 0;
+      goto cleanup;
+    }
 
-  CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy);
-  if (mc != CURLM_OK) {
-    if (mc == CURLM_OUT_OF_MEMORY) {
-      fputs("CURL out of memory. (You are doomed.)\n", stderr);
-      abort();
+    CURLMcode mc = curl_multi_add_handle(pool->multi, newhandle->easy);
+    if (mc != CURLM_OK) {
+      if (mc == CURLM_OUT_OF_MEMORY) {
+        fputs("CURL out of memory. (You are doomed.)\n", stderr);
+        abort();
+      }
+      goto cleanup_multi;
     }
-    goto cleanup;
   }
 
   if (pool->tail) {
@@ -80,20 +99,46 @@ int l2_dlpool_add(l2_dlpool_t *pool, void *user)
     pool->head = pool->tail = newhandle;
   }
 
+  if (!newhandle->easy && !pool->first_waiting) {
+    pool->first_waiting = newhandle;
+  }
+
   return 1;
 
+cleanup_multi:
+  curl_multi_remove_handle(pool->multi, newhandle->easy);
+
 cleanup:
   if (newhandle) {
-    if (newhandle->easy) curl_easy_cleanup(newhandle->easy);
+    if (newhandle->easy) {
+      curl_easy_cleanup(newhandle->easy);
+      ++pool->remaining_handles;
+    }
   }
   free(newhandle);
   return res;
 }
 
+void l2_dlpool__remove_handle(l2_dlpool_t *pool, struct l2_dlpool__handle *handle)
+{
+  if (handle->prev) {
+    handle->prev->next = handle->next;
+  } else {
+    pool->head = handle->next;
+  }
+
+  if (handle->next) {
+    handle->next->prev = handle->prev;
+  } else {
+    pool->tail = handle->prev;
+  }
+}
+
 int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code)
 {
   struct l2_dlpool__handle *handle = NULL;
   int uret = 0;
+  int ret = 1;
   for (struct l2_dlpool__handle *cur = pool->head; cur; cur = cur->next) {
     if (cur->easy == easy) {
       handle = cur;
@@ -104,27 +149,36 @@ int l2_dlpool__download_complete(l2_dlpool_t *pool, CURL *easy, CURLcode code)
   if (handle) {
     uret = (*pool->dlproc)(easy, code, handle->user);
 
-    if (handle->prev) {
-      handle->prev->next = handle->next;
-    } else {
-      pool->head = handle->next;
-    }
-
-    if (handle->next) {
-      handle->next->prev = handle->prev;
-    } else {
-      pool->tail = handle->prev;
-    }
+    l2_dlpool__remove_handle(pool, handle);
 
     free(handle);
     handle = NULL;
   }
 
   curl_multi_remove_handle(pool->multi, easy);
-  curl_easy_cleanup(easy);
+
+  while (pool->first_waiting) {
+    handle = pool->first_waiting;
+    pool->first_waiting = pool->first_waiting->next;
+
+    handle->easy = easy;
+    if (l2_dlpool__call_init(pool, handle) < 0) {
+      l2_dlpool__remove_handle(pool, handle);
+      free(handle);
+      handle = NULL;
+      ret = 0;
+    } else {
+      curl_multi_add_handle(pool->multi, easy);
+      break;
+    }
+  }
+
+  if (!handle) { /* we have not successfully repurposed this handle */
+    curl_easy_cleanup(easy);
+  }
 
   if (uret < 0) return -1;
-  return 0;
+  return ret;
 }
 
 int l2_dlpool_perform(l2_dlpool_t *pool)
@@ -133,6 +187,7 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
   int running = 0;
   int queuelen = 0;
   CURLMsg *msg;
+  int res = 1, ures;
 
   do {
     code = curl_multi_perform(pool->multi, &running);
@@ -142,7 +197,11 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
 
     while ((msg = curl_multi_info_read(pool->multi, &queuelen))) {
       if (msg->msg == CURLMSG_DONE) {
-        if (l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result) < 0) return -1;
+        if ((ures = l2_dlpool__download_complete(pool, msg->easy_handle, msg->data.result)) < 0) {
+          return -1;
+        } else if (ures == 0) {
+          res = 0;
+        }
       }
     }
 
@@ -150,9 +209,9 @@ int l2_dlpool_perform(l2_dlpool_t *pool)
     if (code != CURLM_OK) {
       return -1;
     }
-  } while (running);
+  } while (pool->head);
 
-  return 0;
+  return res;
 }
 
 void l2_dlpool_cleanup(l2_dlpool_t *pool)
diff --git a/src/downloadpool.h b/src/downloadpool.h
index 480fc4f..cce9aea 100644
--- a/src/downloadpool.h
+++ b/src/downloadpool.h
@@ -14,6 +14,7 @@ typedef int (l2_dlpool_download_init_proc_t)(CURL * /*easy*/, void * /*user*/);
 typedef int (l2_dlpool_one_downloaded_proc_t)(CURL * /*easy*/, CURLcode /*code*/, void * /*user*/);
 
 l2_dlpool_t *l2_dlpool_init(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc);
+l2_dlpool_t *l2_dlpool_init2(l2_dlpool_download_init_proc_t *init, l2_dlpool_one_downloaded_proc_t *dlproc, size_t handles);
 
 /* returns >0 for a successful addition, 0 for aborted (init proc returned <0), <0 for error */
 int l2_dlpool_add(l2_dlpool_t *pool, void *user);
diff --git a/src/macros.h b/src/macros.h
index f15cdd7..91924ca 100644
--- a/src/macros.h
+++ b/src/macros.h
@@ -43,6 +43,7 @@
 
 #define L2_URL_META_BASE "https://piston-meta.mojang.com"
 #define L2_URL_META_VERSION_MANIFEST L2_URL_META_BASE "/mc/game/version_manifest_v2.json"
+#define L2_URL_RESOURCES_BASE "https://resources.download.minecraft.net"
 
 #ifdef __GNUC__
 #define L2_FORMAT(_flavor, _stridx, _argidx) __attribute__((format (_flavor, _stridx, _argidx)))
diff --git a/src/version.c b/src/version.c
index 54bdc20..3568cf2 100644
--- a/src/version.c
+++ b/src/version.c
@@ -716,7 +716,7 @@ unsigned l2_version_download_libraries(struct l2_version_library *libs)
   }
 
   if (head != NULL) {
-    if (l2_dlpool_perform(pool) < 0) {
+    if (l2_dlpool_perform(pool) <= 0) {
       CMD_WARN0("Error downloading libraries.");
       res = VERSION_EDOWNLOAD;
     } else {