From 50b42b038a4b0835c0650bb657066cecb87d9291 Mon Sep 17 00:00:00 2001 From: cswimr Date: Wed, 12 Feb 2025 00:53:25 -0600 Subject: [PATCH] feat(sync): init --- bun.lock | 17 +++++- packages/sync/README.md | 15 +++++ packages/sync/package.json | 38 +++++++++++++ packages/sync/src/sync.ts | 110 ++++++++++++++++++++++++++++++++++++ packages/sync/tsconfig.json | 9 +++ 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 packages/sync/README.md create mode 100644 packages/sync/package.json create mode 100644 packages/sync/src/sync.ts create mode 100644 packages/sync/tsconfig.json diff --git a/bun.lock b/bun.lock index 582f0db..6ff0da2 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ }, "packages/parser": { "name": "@packwizjs/parser", - "version": "1.0.4", + "version": "1.1.0", "dependencies": { "@types/semver": "^7.5.8", "semver": "^7.7.1", @@ -28,6 +28,17 @@ "typescript": "^5.0.0", }, }, + "packages/sync": { + "name": "@packwizjs/sync", + "version": "1.0.0", + "dependencies": { + "@packwizjs/parser": "workspace:*", + "murmurhash2": "^0.1.0", + }, + "peerDependencies": { + "typescript": "^5.0.0", + }, + }, }, "packages": { "@colors/colors": ["@colors/colors@1.6.0", "", {}, "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA=="], @@ -64,6 +75,8 @@ "@packwizjs/parser": ["@packwizjs/parser@workspace:packages/parser"], + "@packwizjs/sync": ["@packwizjs/sync@workspace:packages/sync"], + "@rtsao/scc": ["@rtsao/scc@1.1.0", "", {}, "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g=="], "@shikijs/engine-oniguruma": ["@shikijs/engine-oniguruma@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1" } }, "sha512-7iiOx3SG8+g1MnlzZVDYiaeHe7Ez2Kf2HrJzdmGwkRisT7r4rak0e655AcM/tF9JG/kg5fMNYlLLKglbN7gBqA=="], @@ -454,6 +467,8 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "murmurhash2": ["murmurhash2@0.1.0", "", {}, "sha512-k/0g4jE/NRCRqjTeZOzZOfjXs/TqXuV6yl6RfXApoi0io7K7oOJXQ217OymoNDLksmrhCvzcDi/Yj1wOnA57xQ=="], + "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], diff --git a/packages/sync/README.md b/packages/sync/README.md new file mode 100644 index 0000000..b9dcc71 --- /dev/null +++ b/packages/sync/README.md @@ -0,0 +1,15 @@ +# @packwiz/sync + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run sync.ts +``` + +This project was created using `bun init` in bun v1.2.2. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/packages/sync/package.json b/packages/sync/package.json new file mode 100644 index 0000000..26869ed --- /dev/null +++ b/packages/sync/package.json @@ -0,0 +1,38 @@ +{ + "name": "@packwizjs/sync", + "module": "src/sync.ts", + "version": "1.0.0", + "type": "module", + "description": "A directory syncing tool for Minecraft modpacks", + "author": { + "name": "cswimr", + "email": "seaswimmerthefsh@gmail.com" + }, + "repository": { + "url": "https://c.csw.im/GalacticFactory/PackwizJS", + "type": "git" + }, + "license": "GPL-3.0-only", + "files": [ + "src" + ], + "readme": "README.md", + "homepage": "https://packwizjs.csw.im", + "keywords": [ + "packwiz", + "minecraft" + ], + "scripts": {}, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "dependencies": { + "@packwizjs/parser": "workspace:*", + "murmurhash2": "^0.1.0" + }, + "exports": { + ".": { + "import": "./src/sync.ts" + } + } +} diff --git a/packages/sync/src/sync.ts b/packages/sync/src/sync.ts new file mode 100644 index 0000000..2a3375d --- /dev/null +++ b/packages/sync/src/sync.ts @@ -0,0 +1,110 @@ +import { + IndexFileEntry, + Resource, + type Metafile, + type PackwizIndex, + type HashFormat, + doHashesMatch, +} from "@packwizjs/parser"; +import { write } from "bun"; + +function getSaveLocation( + indexFileEntry: IndexFileEntry, + index: PackwizIndex, + metafile?: Metafile, +): Resource { + const cwd = new Resource(process.cwd()); + const diff = index.location.diff(indexFileEntry.file); + const fileDirectory = cwd.join(...diff); + if (metafile) { + return fileDirectory.parent.join(metafile.filename); + } else { + return fileDirectory; + } +} + +async function downloadFile( + hash: string, + hashFormat: HashFormat, + url: Resource, + path: Resource, +) { + const response = await fetch(new URL(url.toString())); + if (!response.ok) { + throw new Error(`Failed to download ${url}: ${response.statusText}`); + } + const arrayBuffer = await response.arrayBuffer(); + if (arrayBuffer.byteLength === 0) { + console.log(`WARNING: Downloaded file is empty: ${url}`); + } + const dataBuffer = Buffer.from(new Uint8Array(arrayBuffer)); + doHashesMatch(hash, hashFormat, dataBuffer); + console.log(`Saving file to ${path.toString()}`); + const fileSize = await write(path.toString(), arrayBuffer); + console.log(`Saved ${fileSize} file to ${path.toString()}`); +} + +/** + * Iterates over the files in a Packwiz index and downloads them concurrently. + * @param index The Packwiz index to iterate over. + * @param concurrencyLimit The maximum number of concurrent downloads. + * @returns A promise that resolves when all files have been downloaded. + */ +export async function iteratePackwizIndex( + index: PackwizIndex, + concurrencyLimit: number = 5, +) { + let currentIndex = 0; + let activeDownloads = 0; + const totalFiles = index.files.length; + + return new Promise((resolve, reject) => { + function downloadNextFile() { + // Resolve if all file entries are downloaded and no active downloads are ongoing + if (currentIndex >= totalFiles && activeDownloads === 0) { + resolve(); + return; + } + + // If there is still file entries to download and there are empty concurrency slots, + // start a new download. + while (activeDownloads < concurrencyLimit && currentIndex < totalFiles) { + const file = index.files[currentIndex++]; + activeDownloads++; + + (async () => { + try { + let saveLocation: Resource; + let url: Resource; + let hash: string; + let hashFormat: HashFormat; + + if (file.metafile) { + const metafile = await file.parse(); + hash = metafile.provider.hash; + hashFormat = metafile.provider.hashFormat; + url = metafile.provider.url; + saveLocation = getSaveLocation(file, index, metafile); + } else { + const diff = index.location.diff(file.file); + hash = file.hash; + hashFormat = file.hashFormat; + url = index.location.parent.join(...diff); + saveLocation = getSaveLocation(file, index); + } + + await downloadFile(hash, hashFormat, url, saveLocation); + } catch (error) { + console.error(`Error downloading file: ${error}`); + reject(error); + return; + } finally { + activeDownloads--; + downloadNextFile(); + } + })(); + } + } + downloadNextFile(); + }); +} diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json new file mode 100644 index 0000000..9e4e31b --- /dev/null +++ b/packages/sync/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + }, + "include": [ + "src/**/*.ts" + ], +}