feat(sync): init

bun.lock

@@ -18,7 +18,7 @@
     },
     "packages/parser": {
       "name": "@packwizjs/parser",
-      "version": "1.0.4",
+      "version": "1.1.0",
       "dependencies": {
         "@types/semver": "^7.5.8",
         "semver": "^7.7.1",
@@ -28,6 +28,17 @@
         "typescript": "^5.0.0",
       },
     },
+    "packages/sync": {
+      "name": "@packwizjs/sync",
+      "version": "1.0.0",
+      "dependencies": {
+        "@packwizjs/parser": "workspace:*",
+        "murmurhash2": "^0.1.0",
+      },
+      "peerDependencies": {
+        "typescript": "^5.0.0",
+      },
+    },
   },
   "packages": {
     "@colors/colors": ["@colors/colors@1.6.0", "", {}, "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA=="],
@@ -64,6 +75,8 @@
 
     "@packwizjs/parser": ["@packwizjs/parser@workspace:packages/parser"],
 
+    "@packwizjs/sync": ["@packwizjs/sync@workspace:packages/sync"],
+
     "@rtsao/scc": ["@rtsao/scc@1.1.0", "", {}, "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g=="],
 
     "@shikijs/engine-oniguruma": ["@shikijs/engine-oniguruma@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1" } }, "sha512-7iiOx3SG8+g1MnlzZVDYiaeHe7Ez2Kf2HrJzdmGwkRisT7r4rak0e655AcM/tF9JG/kg5fMNYlLLKglbN7gBqA=="],
@@ -454,6 +467,8 @@
 
     "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
 
+    "murmurhash2": ["murmurhash2@0.1.0", "", {}, "sha512-k/0g4jE/NRCRqjTeZOzZOfjXs/TqXuV6yl6RfXApoi0io7K7oOJXQ217OymoNDLksmrhCvzcDi/Yj1wOnA57xQ=="],
+
     "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="],
 
     "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],

packages/sync/README.md

@@ -0,0 +1,15 @@
+# @packwiz/sync
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run sync.ts
+```
+
+This project was created using `bun init` in bun v1.2.2. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

packages/sync/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@packwizjs/sync",
+  "module": "src/sync.ts",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "A directory syncing tool for Minecraft modpacks",
+  "author": {
+    "name": "cswimr",
+    "email": "seaswimmerthefsh@gmail.com"
+  },
+  "repository": {
+    "url": "https://c.csw.im/GalacticFactory/PackwizJS",
+    "type": "git"
+  },
+  "license": "GPL-3.0-only",
+  "files": [
+    "src"
+  ],
+  "readme": "README.md",
+  "homepage": "https://packwizjs.csw.im",
+  "keywords": [
+    "packwiz",
+    "minecraft"
+  ],
+  "scripts": {},
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@packwizjs/parser": "workspace:*",
+    "murmurhash2": "^0.1.0"
+  },
+  "exports": {
+    ".": {
+      "import": "./src/sync.ts"
+    }
+  }
+}

packages/sync/src/sync.ts

@@ -0,0 +1,110 @@
+import {
+  IndexFileEntry,
+  Resource,
+  type Metafile,
+  type PackwizIndex,
+  type HashFormat,
+  doHashesMatch,
+} from "@packwizjs/parser";
+import { write } from "bun";
+
+function getSaveLocation(
+  indexFileEntry: IndexFileEntry,
+  index: PackwizIndex,
+  metafile?: Metafile,
+): Resource {
+  const cwd = new Resource(process.cwd());
+  const diff = index.location.diff(indexFileEntry.file);
+  const fileDirectory = cwd.join(...diff);
+  if (metafile) {
+    return fileDirectory.parent.join(metafile.filename);
+  } else {
+    return fileDirectory;
+  }
+}
+
+async function downloadFile(
+  hash: string,
+  hashFormat: HashFormat,
+  url: Resource,
+  path: Resource,
+) {
+  const response = await fetch(new URL(url.toString()));
+  if (!response.ok) {
+    throw new Error(`Failed to download ${url}: ${response.statusText}`);
+  }
+  const arrayBuffer = await response.arrayBuffer();
+  if (arrayBuffer.byteLength === 0) {
+    console.log(`WARNING: Downloaded file is empty: ${url}`);
+  }
+  const dataBuffer = Buffer.from(new Uint8Array(arrayBuffer));
+  doHashesMatch(hash, hashFormat, dataBuffer);
+  console.log(`Saving file to ${path.toString()}`);
+  const fileSize = await write(path.toString(), arrayBuffer);
+  console.log(`Saved ${fileSize} file to ${path.toString()}`);
+}
+
+/**
+ * Iterates over the files in a Packwiz index and downloads them concurrently.
+ * @param index The Packwiz index to iterate over.
+ * @param concurrencyLimit The maximum number of concurrent downloads.
+ * @returns A promise that resolves when all files have been downloaded.
+ */
+export async function iteratePackwizIndex(
+  index: PackwizIndex,
+  concurrencyLimit: number = 5,
+) {
+  let currentIndex = 0;
+  let activeDownloads = 0;
+  const totalFiles = index.files.length;
+
+  return new Promise<void>((resolve, reject) => {
+    function downloadNextFile() {
+      // Resolve if all file entries are downloaded and no active downloads are ongoing
+      if (currentIndex >= totalFiles && activeDownloads === 0) {
+        resolve();
+        return;
+      }
+
+      // If there is still file entries to download and there are empty concurrency slots,
+      // start a new download.
+      while (activeDownloads < concurrencyLimit && currentIndex < totalFiles) {
+        const file = index.files[currentIndex++];
+        activeDownloads++;
+
+        (async () => {
+          try {
+            let saveLocation: Resource;
+            let url: Resource;
+            let hash: string;
+            let hashFormat: HashFormat;
+
+            if (file.metafile) {
+              const metafile = await file.parse();
+              hash = metafile.provider.hash;
+              hashFormat = metafile.provider.hashFormat;
+              url = metafile.provider.url;
+              saveLocation = getSaveLocation(file, index, metafile);
+            } else {
+              const diff = index.location.diff(file.file);
+              hash = file.hash;
+              hashFormat = file.hashFormat;
+              url = index.location.parent.join(...diff);
+              saveLocation = getSaveLocation(file, index);
+            }
+
+            await downloadFile(hash, hashFormat, url, saveLocation);
+          } catch (error) {
+            console.error(`Error downloading file: ${error}`);
+            reject(error);
+            return;
+          } finally {
+            activeDownloads--;
+            downloadNextFile();
+          }
+        })();
+      }
+    }
+    downloadNextFile();
+  });
+}

packages/sync/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+}