1 files changed, 506 insertions, 3 deletions

diff --git a/src/runtime.c b/src/runtime.c
index b20353c..b811519 100644
--- a/src/runtime.c
+++ b/src/runtime.c
@@ -4,6 +4,7 @@
 #include "macros.h"
 #include "l2su.h"
 #include "endian.h"
+#include "downloadpool.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -12,10 +13,12 @@
 #include <stdio.h>
 #include <stdint.h>
 #include <curl/curl.h>
+#include <sys/stat.h>
 #include <unistd.h>
 #include <time.h>
 #include <inttypes.h>
 #include <ftw.h>
+#include <lzma.h>
 
 #define L2_RUNTIME__MANIFEST_EXP_TIME ((time_t)120)
 
@@ -361,7 +364,7 @@ int l2_runtime__save_component_manifest_info(const char *compname, const char *v
 
 int l2_runtime__check_runtime_ftw(const char *fpath, const struct stat *sb, int typeflag, struct l2_ftw_data *ftwbuf);
 
-int l2_runtime__ensure_runtime_files(json_t *jfiles);
+int l2_runtime__ensure_runtime_files(const char *basepath, json_t *jfiles);
 
 int l2_runtime_install_component(json_t *manifest, const char *component)
 {
@@ -428,7 +431,7 @@ int l2_runtime_install_component(json_t *manifest, const char *component)
     goto cleanup;
   }
 
-  if (l2_runtime__ensure_runtime_files(jfiles) < 0) {
+  if (l2_runtime__ensure_runtime_files(jrepath, jfiles) < 0) {
     goto cleanup;
   }
 
@@ -440,6 +443,500 @@ cleanup:
   return -1;
 }
 
+struct l2_runtime__file_info {
+  /* set at init */
+  char *lzmaurl, *rawurl;
+  char *path;
+  l2_sha1_digest_t expect_digest;
+  size_t expect_size;
+
+  l2_sha1_state_t digest_state;
+  size_t nread_total;
+
+  FILE *ofile;
+
+  bool lzma_active;
+  lzma_stream lz;
+
+  struct l2_runtime__file_info *next;
+};
+
+void l2_runtime__file_info_cleanup(struct l2_runtime__file_info *info)
+{
+  free(info->lzmaurl);
+  free(info->rawurl);
+
+  if (info->ofile) {
+    fclose(info->ofile);
+    if (info->path && unlink(info->path) < 0 && errno != ENOENT) {
+      CMD_WARN("Failed to delete %s: %s", info->path, strerror(errno));
+    }
+  }
+
+  free(info->path);
+
+  if (info->lzma_active) {
+    lzma_end(&info->lz);
+  }
+}
+
+int l2_runtime__ensure_one_file(const char *filename, json_t *file, struct l2_runtime__file_info **ptail)
+{
+  l2_sha1_digest_t expect_digest;
+  const char *expect_digest_str;
+  json_int_t expect_size;
+  struct l2_runtime__file_info *info = NULL;
+
+  const char *raw_url = NULL;
+
+  json_t *lzma_download = NULL;
+  if (json_unpack(file, "{s:{s:{s:s, s:I, s?:s}, s?:o}}", "downloads", "raw", "sha1", &expect_digest_str, "size", &expect_size, "url", &raw_url, "lzma", &lzma_download) < 0) {
+    CMD_DEBUG0("invalid format for runtime manifest: couldn't get downloads");
+    return -1;
+  }
+
+  if (l2_sha1_digest_from_hex(&expect_digest, expect_digest_str) < 0) {
+    CMD_DEBUG("invalid format for runtime manifest: sha1 hash is nonsensical %s", expect_digest_str);
+    return -1;
+  }
+
+  FILE *fp = fopen(filename, "rb");
+  if (!fp) {
+    if (errno == ENOENT) {
+      goto ok_we_have_to_download_the_file;
+    } else {
+      CMD_WARN("Cannot open %s for reading: %s", filename, strerror(errno));
+      return -1;
+    }
+  }
+
+  int integres = l2_launcher_check_integrity(fp, &expect_digest, (size_t)expect_size);
+  fclose(fp);
+  fp = NULL;
+
+  if (integres < 0) {
+    CMD_WARN("Failed to check file integrity for %s.", filename);
+    return -1;
+  } else if (integres) {
+    CMD_DEBUG("Not downloading %s, integrity info matches :)", filename);
+    return 0;
+  }
+
+ok_we_have_to_download_the_file:
+  info = calloc(1, sizeof(struct l2_runtime__file_info));
+  if (!info) {
+    CMD_DEBUG("allocation failed! %s %s", filename, strerror(errno));
+    return -1;
+  }
+
+  const char *lzma_url = NULL;
+  if (lzma_download && json_unpack(lzma_download, "{s:s}", "url", &lzma_url) < 0) {
+    CMD_DEBUG0("invalid format for lzma download: no url");
+    goto cleanup;
+  }
+
+  if (lzma_url) {
+    info->lzmaurl = strdup(lzma_url);
+    if (!info->lzmaurl) goto cleanup;
+  } else {
+    if (!raw_url) {
+      CMD_WARN("No download for %s!", filename);
+      goto cleanup;
+    }
+
+    info->rawurl = strdup(raw_url);
+  }
+
+  info->path = strdup(filename);
+  if (!info->path) {
+    goto cleanup;
+  }
+
+  info->expect_size = (size_t)expect_size;
+  l2_sha1_digest_copy(&info->expect_digest, &expect_digest);
+
+  if (*ptail) {
+    (*ptail)->next = info;
+  }
+  *ptail = info;
+  return 0;
+
+cleanup:
+  l2_runtime__file_info_cleanup(info);
+  free(info);
+  return -1;
+}
+
+int l2_runtime__ensure_one_link(const char *filename, json_t *file)
+{
+  /* TODO */
+  const char *target;
+  size_t targetlen;
+  if (json_unpack(file, "{s:s%}", "target", &target, &targetlen) < 0) {
+    CMD_DEBUG0("bad link");
+    return -1;
+  }
+
+  char *buf = alloca(targetlen + 1);
+  ssize_t nread;
+
+  if ((nread = readlink(filename, buf, targetlen + 1)) < 0) {
+    if (errno == ENOENT) {
+      goto set_link;
+    }
+    CMD_WARN("Could not read link at %s: %s", filename, strerror(errno));
+    return -1;
+  }
+
+  buf[targetlen] = '\0';
+  if (targetlen == (size_t)nread && !strcmp(buf, target)) {
+    CMD_DEBUG("link %s is good", filename);
+    return 0;
+  }
+
+  if (unlink(filename) < 0) {
+    CMD_WARN("Failed to delete symlink %s: %s", filename, strerror(errno));
+    return -1;
+  }
+
+set_link:
+  if (symlink(target, filename) < 0) {
+    CMD_WARN("Failed to create symlink %s to %s: %s", filename, target, strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+
+int l2_runtime__component_update_file(const char *fname, json_t *file, const char *basepath, struct l2_runtime__file_info **ptail)
+{
+  char *path;
+  size_t pathsz;
+  L2_ASPRINTF(path, pathsz, "%s/%s", basepath, fname);
+
+  const char *typestr;
+  if (json_unpack(file, "{s:s}", "type", &typestr) < 0) {
+    CMD_DEBUG0("bad type info or something idk");
+    return -1;
+  }
+
+  mode_t exptype = 0;
+  if (!strcmp(typestr, "file")) {
+    exptype = S_IFREG;
+  } else if (!strcmp(typestr, "directory")) {
+    exptype = S_IFDIR;
+  } else if (!strcmp(typestr, "link")) {
+    exptype = S_IFLNK;
+  } else {
+    CMD_DEBUG("unknown file type %s, %s", path, typestr);
+    return -1;
+  }
+
+  struct stat sp;
+  bool exists = true;
+  if (lstat(path, &sp) < 0) {
+    if (errno == ENOENT) {
+      exists = false;
+    } else {
+      CMD_WARN("could not stat %s: %s", path, strerror(errno));
+      return -1;
+    }
+  }
+
+  if (exists && (sp.st_mode & S_IFMT) != exptype) { /* should never happen besides by a race condition */
+    CMD_WARN("File %s has wrong type! (should be %s (%o), got %o)", path, typestr, exptype, sp.st_mode & S_IFMT);
+    return -1;
+  }
+
+  switch (exptype) {
+    case S_IFDIR:
+      if (!exists && mkdir(path, 0755) < 0) {
+        CMD_WARN("Failed to make directory %s: %s", path, strerror(errno));
+        return -1;
+      }
+      return 0;
+    case S_IFREG:
+      return l2_runtime__ensure_one_file(path, file, ptail);
+    case S_IFLNK:
+      return l2_runtime__ensure_one_link(path, file);
+  }
+
+  CMD_MSG0("bug", "expected file type was not file, link, or directory!!!!");
+  abort();
+}
+
+const char *l2_runtime__get_lzma_error(lzma_ret lzret)
+{
+  switch (lzret) {
+    case LZMA_OK:
+    case LZMA_STREAM_END:
+      return NULL;
+    case LZMA_MEM_ERROR:
+      return "out of memory";
+    case LZMA_MEMLIMIT_ERROR:
+      return "memory limit reached";
+    case LZMA_FORMAT_ERROR:
+      return "LZMA/XZ format error";
+    case LZMA_OPTIONS_ERROR:
+      return "invalid usage";
+    case LZMA_BUF_ERROR:
+      return "file is truncated or corrupt";
+    case LZMA_DATA_ERROR:
+      return "file is corrupt";
+    default:
+      return "unknown error";
+  }
+}
+
+size_t l2_runtime__dl_lzma_callback(char *buf, size_t size, size_t nmemb, void *user)
+{
+  struct l2_runtime__file_info *info = user;
+  size_t realsz = size * nmemb;
+  info->lz.total_in = 0;
+
+  uint8_t outbuf[4096];
+  info->lz.next_in = (const uint8_t *)buf;
+  info->lz.avail_in = realsz;
+  do {  
+    info->lz.next_out = outbuf;
+    info->lz.avail_out = sizeof(outbuf);
+
+    lzma_ret lzret = lzma_code(&info->lz, LZMA_RUN);
+    const char *errstr = l2_runtime__get_lzma_error(lzret);
+
+    if (errstr) {
+      CMD_ERROR("Error downloading %s: %s (error %u)", info->path, errstr, lzret);
+      return CURL_WRITEFUNC_ERROR;
+    }
+
+    size_t ndecoded = sizeof(outbuf) - info->lz.avail_out;
+
+    if (!ndecoded) return realsz;
+
+    info->nread_total += ndecoded;
+    l2_sha1_update(&info->digest_state, outbuf, ndecoded);
+
+    errno = 0;
+    if (fwrite(outbuf, 1, ndecoded, info->ofile) < ndecoded) {
+      CMD_WARN("Partial write %s: %s", info->path, strerror(errno));
+      return CURL_WRITEFUNC_ERROR;
+    }
+  } while (info->lz.avail_in);
+
+  return realsz;
+}
+
+size_t l2_runtime__dl_raw_callback(char *buf, size_t size, size_t nmemb, void *user)
+{
+  struct l2_runtime__file_info *info = user;
+  size_t realsz = size * nmemb;
+
+  info->nread_total += realsz;
+  l2_sha1_update(&info->digest_state, buf, realsz);
+
+  errno = 0;
+  if (fwrite(buf, size, nmemb, info->ofile) < nmemb) {
+    CMD_WARN("Partial write %s: %s", info->path, strerror(errno));
+    return CURL_WRITEFUNC_ERROR;
+  }
+
+  return realsz;
+}
+
+int l2_runtime__dl_init(CURL *curl, void *user)
+{
+  struct l2_runtime__file_info *info = user;
+
+  info->ofile = fopen(info->path, "wb");
+  if (!info->ofile) {
+    CMD_WARN("Failed to open %s for writing: %s", info->path, strerror(errno));
+    return -1;
+  }
+
+  l2_sha1_init(&info->digest_state);
+
+  if (info->lzmaurl) {
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_ret lzret;
+    memcpy(&info->lz, &strm, sizeof(lzma_stream));
+
+    switch (lzret = lzma_auto_decoder(&info->lz, UINT64_MAX, 0)) {
+      case LZMA_OK:
+        break;
+      case LZMA_MEM_ERROR:
+        CMD_WARN("Failed to initialize LZMA stream for %s: out of memory", info->path);
+        return -1;
+      case LZMA_OPTIONS_ERROR:
+        CMD_WARN("Failed to initialize LZMA stream for %s: invalid argument", info->path);
+        return -1;
+      case LZMA_PROG_ERROR:
+        CMD_WARN("Failed to initialize LZMA stream for %s: bug", info->path);
+        return -1;
+      default:
+        CMD_WARN("Failed to initialize LZMA stream for %s: unknown error %u", info->path, lzret);
+        return -1;
+    }
+
+    curl_easy_setopt(curl, CURLOPT_URL, info->lzmaurl);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &l2_runtime__dl_lzma_callback);
+    info->lzma_active = true;
+  } else {
+    curl_easy_setopt(curl, CURLOPT_URL, info->rawurl);
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &l2_runtime__dl_raw_callback);
+    info->lzma_active = false;
+  }
+
+  curl_easy_setopt(curl, CURLOPT_USERAGENT, L2_USER_AGENT);
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, user);
+
+  return 0;
+}
+
+int l2_runtime__dl_complete(CURL *curl, CURLcode code, void *user)
+{
+  struct l2_runtime__file_info *info = user;
+
+  if (code != CURLE_OK) {
+    CMD_WARN("Download for %s failed: CURL error: %s", info->path, curl_easy_strerror(code));
+    return -1;
+  }
+
+  long httpres = 0;
+  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpres);
+  if (httpres / 100 != 2) {
+    CMD_WARN("Download for %s failed: server responded with %lu", info->path, httpres);
+    return -1;
+  }
+
+  if (info->lzma_active) {
+    uint8_t outbuf[4096];
+    info->lz.next_in = NULL;
+    info->lz.avail_in = 0;
+
+    lzma_ret lzret;
+    while (true) {
+      info->lz.next_out = outbuf;
+      info->lz.avail_out = sizeof(outbuf);
+
+      lzret = lzma_code(&info->lz, LZMA_FINISH);
+      const char *errstr = l2_runtime__get_lzma_error(lzret);
+
+      if (errstr) {
+        CMD_WARN("Download for %s failed: LZMA error: %s (error %u)", info->path, errstr, lzret);
+        return -1;
+      }
+
+      size_t ndecoded = sizeof(outbuf) - info->lz.avail_out;
+      if (!ndecoded) break;
+
+      l2_sha1_update(&info->digest_state, outbuf, ndecoded);
+      info->nread_total += ndecoded;
+
+      errno = 0;
+      if (fwrite(outbuf, 1, ndecoded, info->ofile) < ndecoded) {
+        CMD_WARN("Download for %s failed: partial write: %s", info->path, strerror(errno));
+        return -1;
+      }
+
+      if (lzret == LZMA_STREAM_END) break;
+    }
+
+    lzma_end(&info->lz);
+    info->lzma_active = false;
+  }
+
+  fclose(info->ofile);
+  info->ofile = NULL;
+
+  l2_sha1_digest_t rddg;
+  l2_sha1_finalize(&info->digest_state, &rddg);
+  if (l2_sha1_digest_compare(&rddg, &info->expect_digest)) {
+    char expstr[L2_SHA1_HEX_STRLEN + 1], gotstr[L2_SHA1_HEX_STRLEN + 1];
+    l2_sha1_digest_to_hex(&info->expect_digest, expstr);
+    l2_sha1_digest_to_hex(&rddg, gotstr);
+    CMD_WARN("Download for %s failed: SHA1 mismatch (expected %s, got %s)", info->path, expstr, gotstr);
+    goto cleanup_unlink;
+  }
+
+  if (info->nread_total != info->expect_size) {
+    CMD_WARN("Download for %s failed: size mismatch (expected %zu bytes, got %zu bytes)", info->path, info->expect_size, info->nread_total);
+    goto cleanup_unlink;
+  }
+
+  return 0;
+
+cleanup_unlink:
+  if (unlink(info->path) < 0) {
+    CMD_WARN("Failed to delete %s: %s", info->path, strerror(errno));
+  }
+  return -1;
+}
+
+int l2_runtime__ensure_runtime_files(const char *basepath, json_t *jfiles)
+{
+  int ret = 0;
+  struct l2_runtime__file_info *info_head, *info_tail;
+  info_head = info_tail = NULL;
+  l2_dlpool_t *dlpool = NULL;
+
+  char *fullpath = NULL;
+  size_t len = 0; 
+
+  json_t *file;
+  const char *fname;
+
+  json_object_foreach(jfiles, fname, file) {
+    if (!l2_launcher_sprintf_resize(&fullpath, &len, "%s/%s", basepath, fname)) {
+      ret = -1;
+      goto cleanup;
+    }
+
+    if (l2_runtime__component_update_file(fname, file, basepath, &info_tail) < 0) {
+      ret = -1;
+      goto cleanup;
+    }
+
+    if (info_tail && !info_head) info_head = info_tail;
+  }
+
+  if (!info_head) {
+    CMD_INFO("No files to download for %s.", basepath);
+    ret = 0;
+    goto cleanup;
+  }
+ 
+  dlpool = l2_dlpool_init(&l2_runtime__dl_init, &l2_runtime__dl_complete);
+  if (!dlpool) {
+    ret = -1;
+    goto cleanup;
+  }
+
+  for (struct l2_runtime__file_info *cur = info_head; cur; cur = cur->next) {
+    if (l2_dlpool_add(dlpool, cur) <= 0) {
+      ret = -1;
+      goto cleanup;
+    }
+  }
+
+  if (l2_dlpool_perform(dlpool) < 0) {
+    ret = -1;
+  } else {
+    ret = 0;
+  }
+
+cleanup:
+  free(fullpath);
+  if (dlpool) l2_dlpool_cleanup(dlpool);
+
+  for (struct l2_runtime__file_info *temp; info_head; info_head = temp) {
+    temp = info_head->next;
+
+    l2_runtime__file_info_cleanup(info_head);
+    free(info_head);
+  }
+  return ret;
+}
+
 int l2_runtime__check_runtime_ftw(const char *fpath, const struct stat *sb, int typeflag, struct l2_ftw_data *ftw)
 {
   L2_UNUSED(sb);
@@ -492,7 +989,13 @@ int l2_runtime__check_runtime_ftw(const char *fpath, const struct stat *sb, int
       return -1;
     }
 
-    unsigned desired_mode = (sb->st_mode & 07777) | 0111;
+    unsigned desired_mode;
+    if (executable) {
+      desired_mode = (sb->st_mode & 07777) | 0111;
+    } else {
+      desired_mode = (sb->st_mode & 07777) & ~0111;
+    }
+
     if ((sb->st_mode & 07777) != desired_mode && chmod(fpath, desired_mode) < 0) {
       CMD_WARN("Failed chmod(%s, %#o): %s", fpath, desired_mode, strerror(errno));
       return -1;