diff --git a/packages/sync/src/index.ts b/packages/sync/src/index.ts index 6e79ec5..a820e06 100644 --- a/packages/sync/src/index.ts +++ b/packages/sync/src/index.ts @@ -1 +1 @@ -export { iteratePackwizIndex, formatBytes } from "./sync"; +export { iteratePackwizIndex, readStateFile, formatBytes } from "./sync"; diff --git a/packages/sync/src/sync.ts b/packages/sync/src/sync.ts index 0e7afbb..9efdd52 100644 --- a/packages/sync/src/sync.ts +++ b/packages/sync/src/sync.ts @@ -1,5 +1,5 @@ import { - IndexFileEntry, + Packwiz, Resource, type Metafile, type PackwizIndex, @@ -10,13 +10,26 @@ import { } from "@packwizjs/parser"; import { write } from "bun"; +interface HashInfo { + hashFormat: HashFormat; + hash: string; +} + +interface State { + stateVersion: number; + currentVersion: string; + hashes: { + [fileName: string]: HashInfo; + }; +} + function getSaveLocation( - indexFileEntry: IndexFileEntry, + path: Resource, index: PackwizIndex, metafile?: Metafile, ): Resource { const cwd = new Resource(process.cwd()); - const diff = index.location.diff(indexFileEntry.file); + const diff = index.location.diff(path); const fileDirectory = cwd.join(...diff); if (metafile) { return fileDirectory.parent.join(metafile.filename); @@ -25,6 +38,50 @@ function getSaveLocation( } } +async function writeStateFile( + packwiz: Packwiz, + hashes: { [fileName: string]: HashInfo }, +) { + const stateFile = packwiz.index.location.parent.join(".packwizjs-state.json"); + const state: State = { + stateVersion: 1, + currentVersion: packwiz.version, + hashes: hashes, + }; + // for (const file of packwiz.index.files) { + // state.hashes[packwiz.index.location.diff(file.file).join("/")] = { + // hashFormat: file.hashFormat, + // hash: file.hash, + // }; + // } + const saveLocation = getSaveLocation(stateFile, packwiz.index); + await write(saveLocation.toString(), JSON.stringify(state, null)); +} + +/** + * Reads the state file from the specified location. + * @param stateFile - The path to the state file. Defaults to ".packwizjs-state.json" in the current working directory. + * @returns A Promise containing the parsed state object. + */ +export async function readStateFile( + stateFile: Resource = new Resource(process.cwd()).join( + ".packwizjs-state.json", + ), +): Promise { + // When adding a new state version, do NOT change the name of the stateVersion key!!! That key is NOT versioned. + const json = JSON.parse(await stateFile.fetchContents()); + const stateVersion: number = json["stateVersion"]; + switch (stateVersion) { + case 1: + const state: State = { ...json }; + return state; + default: + throw new Error( + `Unsupported state version: ${stateVersion}. Please update '@packwizjs/sync' to use newer state file versions.`, + ); + } +} + async function downloadFile( hash: string, hashFormat: HashFormat, @@ -65,18 +122,27 @@ export function formatBytes(bytes: number, decimals: number = 2) { * Iterates over the files in a Packwiz index and downloads them concurrently. * @param index The Packwiz index to iterate over. * @param side The side to download files for (e.g., "client", "server", or "both"). + * @param path The path to save the downloaded files. Defaults to the current working directory. * @param concurrencyLimit The maximum number of concurrent downloads. * @returns A promise that resolves when all files have been downloaded. */ export async function iteratePackwizIndex( - index: PackwizIndex, + packwiz: Packwiz, side: Side, + path: Resource = new Resource(process.cwd()), concurrencyLimit: number = 5, ) { let currentIndex = 0; let activeDownloads = 0; - const totalFiles = index.files.length; + const totalFiles = packwiz.index.files.length; + const stateFile = path.join(".packwizjs-state.json"); + let state: State | undefined; + if (await stateFile.exists()) { + state = await readStateFile(stateFile); + } + console.log(state); side = side.toLowerCase() as Side; + const hashes: { [fileName: string]: HashInfo } = {}; return new Promise((resolve, reject) => { function downloadNextFile() { @@ -89,7 +155,7 @@ export async function iteratePackwizIndex( // If there is still file entries to download and there are empty concurrency slots, // start a new download. while (activeDownloads < concurrencyLimit && currentIndex < totalFiles) { - const file = index.files[currentIndex++]; + const file = packwiz.index.files[currentIndex++]; activeDownloads++; (async () => { @@ -119,18 +185,45 @@ export async function iteratePackwizIndex( hash = metafile.provider.hash; hashFormat = metafile.provider.hashFormat; url = metafile.provider.url; - saveLocation = getSaveLocation(file, index, metafile); + saveLocation = getSaveLocation( + file.file, + packwiz.index, + metafile, + ); } else { // we don't check non-metafiles' Side because packwiz doesn't store metadata for whether or not to download them on client / server // so instead, always download them on both sides (assume Side.Both) - const diff = index.location.diff(file.file); + const diff = packwiz.index.location.diff(file.file); hash = file.hash; hashFormat = file.hashFormat; - url = index.location.parent.join(...diff); - saveLocation = getSaveLocation(file, index); + url = packwiz.index.location.parent.join(...diff); + saveLocation = getSaveLocation(file.file, packwiz.index); } - await downloadFile(hash, hashFormat, url, saveLocation); + if (state) { + let stateHash: HashInfo | undefined; + // TODO: implement hash checking for files downloaded via metafile providers + if (!file.metafile) { + stateHash = + state.hashes[ + packwiz.index.location.diff(file.file).join("/") + ]; + } + if (stateHash?.hash && stateHash.hash === hash) { + console.log( + `Skipping already downloaded file ${file.file.toString()}`, + ); + } else { + await downloadFile(hash, hashFormat, url, saveLocation); + } + } else { + await downloadFile(hash, hashFormat, url, saveLocation); + } + + hashes[packwiz.index.location.diff(file.file).join("/")] = { + hashFormat: hashFormat, + hash: hash, + }; } catch (error) { console.error(`Error downloading file: ${error}`); reject(error); @@ -143,5 +236,5 @@ export async function iteratePackwizIndex( } } downloadNextFile(); - }); + }).then(() => writeStateFile(packwiz, hashes)); }