feat(sync): add (wip) state file

flake.nix

@@ -24,6 +24,7 @@
           ];
           packages = with pkgs; [
             packwiz
+            hyperfine
           ];
 
           shellHook = ''

package.json

@@ -12,7 +12,7 @@
   "scripts": {
     "docs": "typedoc --options typedoc.jsonc",
     "format": "bun prettier --write '**/*.{ts,json,md}'",
-    "prepare": "husky || true"
+    "prepare": "bunx husky || true"
   },
   "devDependencies": {
     "@getmeli/cli": "1.2.0",

packages/sync/src/index.ts

@@ -1 +1 @@
-export { iteratePackwizIndex, formatBytes } from "./sync";
+export { iteratePackwizIndex, readStateFile, formatBytes } from "./sync";

packages/sync/src/sync.ts

@@ -1,5 +1,5 @@
 import {
-  IndexFileEntry,
+  Packwiz,
   Resource,
   type Metafile,
   type PackwizIndex,
@@ -10,13 +10,26 @@ import {
 } from "@packwizjs/parser";
 import { write } from "bun";
 
+interface HashInfo {
+  hashFormat: HashFormat;
+  hash: string;
+}
+
+interface State {
+  stateVersion: number;
+  currentVersion: string;
+  hashes: {
+    [fileName: string]: HashInfo;
+  };
+}
+
 function getSaveLocation(
-  indexFileEntry: IndexFileEntry,
+  path: Resource,
   index: PackwizIndex,
   metafile?: Metafile,
 ): Resource {
   const cwd = new Resource(process.cwd());
-  const diff = index.location.diff(indexFileEntry.file);
+  const diff = index.location.diff(path);
   const fileDirectory = cwd.join(...diff);
   if (metafile) {
     return fileDirectory.parent.join(metafile.filename);
@@ -25,6 +38,50 @@ function getSaveLocation(
   }
 }
 
+async function writeStateFile(
+  packwiz: Packwiz,
+  hashes: { [fileName: string]: HashInfo },
+) {
+  const stateFile = packwiz.index.location.parent.join(".packwizjs-state.json");
+  const state: State = {
+    stateVersion: 1,
+    currentVersion: packwiz.version,
+    hashes: hashes,
+  };
+  // for (const file of packwiz.index.files) {
+  //   state.hashes[packwiz.index.location.diff(file.file).join("/")] = {
+  //     hashFormat: file.hashFormat,
+  //     hash: file.hash,
+  //   };
+  // }
+  const saveLocation = getSaveLocation(stateFile, packwiz.index);
+  await write(saveLocation.toString(), JSON.stringify(state, null));
+}
+
+/**
+ * Reads the state file from the specified location.
+ * @param stateFile - The path to the state file. Defaults to ".packwizjs-state.json" in the current working directory.
+ * @returns A Promise containing the parsed state object.
+ */
+export async function readStateFile(
+  stateFile: Resource = new Resource(process.cwd()).join(
+    ".packwizjs-state.json",
+  ),
+): Promise<State> {
+  // When adding a new state version, do NOT change the name of the stateVersion key!!! That key is NOT versioned.
+  const json = JSON.parse(await stateFile.fetchContents());
+  const stateVersion: number = json["stateVersion"];
+  switch (stateVersion) {
+    case 1:
+      const state: State = { ...json };
+      return state;
+    default:
+      throw new Error(
+        `Unsupported state version: ${stateVersion}. Please update '@packwizjs/sync' to use newer state file versions.`,
+      );
+  }
+}
+
 async function downloadFile(
   hash: string,
   hashFormat: HashFormat,
@@ -65,18 +122,27 @@ export function formatBytes(bytes: number, decimals: number = 2) {
  * Iterates over the files in a Packwiz index and downloads them concurrently.
  * @param index The Packwiz index to iterate over.
  * @param side The side to download files for (e.g., "client", "server", or "both").
+ * @param path The path to save the downloaded files. Defaults to the current working directory.
  * @param concurrencyLimit The maximum number of concurrent downloads.
  * @returns A promise that resolves when all files have been downloaded.
  */
 export async function iteratePackwizIndex(
-  index: PackwizIndex,
+  packwiz: Packwiz,
   side: Side,
+  path: Resource = new Resource(process.cwd()),
   concurrencyLimit: number = 5,
 ) {
   let currentIndex = 0;
   let activeDownloads = 0;
-  const totalFiles = index.files.length;
+  const totalFiles = packwiz.index.files.length;
+  const stateFile = path.join(".packwizjs-state.json");
+  let state: State | undefined;
+  if (await stateFile.exists()) {
+    state = await readStateFile(stateFile);
+  }
+  console.log(state);
   side = side.toLowerCase() as Side;
+  const hashes: { [fileName: string]: HashInfo } = {};
 
   return new Promise<void>((resolve, reject) => {
     function downloadNextFile() {
@@ -89,7 +155,7 @@ export async function iteratePackwizIndex(
       // If there is still file entries to download and there are empty concurrency slots,
       // start a new download.
       while (activeDownloads < concurrencyLimit && currentIndex < totalFiles) {
-        const file = index.files[currentIndex++];
+        const file = packwiz.index.files[currentIndex++];
         activeDownloads++;
 
         (async () => {
@@ -119,18 +185,45 @@ export async function iteratePackwizIndex(
               hash = metafile.provider.hash;
               hashFormat = metafile.provider.hashFormat;
               url = metafile.provider.url;
-              saveLocation = getSaveLocation(file, index, metafile);
+              saveLocation = getSaveLocation(
+                file.file,
+                packwiz.index,
+                metafile,
+              );
             } else {
               // we don't check non-metafiles' Side because packwiz doesn't store metadata for whether or not to download them on client / server
               // so instead, always download them on both sides (assume Side.Both)
-              const diff = index.location.diff(file.file);
+              const diff = packwiz.index.location.diff(file.file);
               hash = file.hash;
               hashFormat = file.hashFormat;
-              url = index.location.parent.join(...diff);
+              url = packwiz.index.location.parent.join(...diff);
-              saveLocation = getSaveLocation(file, index);
+              saveLocation = getSaveLocation(file.file, packwiz.index);
             }
 
+            if (state) {
+              let stateHash: HashInfo | undefined;
+              // TODO: implement hash checking for files downloaded via metafile providers
+              if (!file.metafile) {
+                stateHash =
+                  state.hashes[
+                    packwiz.index.location.diff(file.file).join("/")
+                  ];
+              }
+              if (stateHash?.hash && stateHash.hash === hash) {
+                console.log(
+                  `Skipping already downloaded file ${file.file.toString()}`,
+                );
+              } else {
                 await downloadFile(hash, hashFormat, url, saveLocation);
+              }
+            } else {
+              await downloadFile(hash, hashFormat, url, saveLocation);
+            }
+
+            hashes[packwiz.index.location.diff(file.file).join("/")] = {
+              hashFormat: hashFormat,
+              hash: hash,
+            };
           } catch (error) {
             console.error(`Error downloading file: ${error}`);
             reject(error);
@@ -143,5 +236,5 @@ export async function iteratePackwizIndex(
       }
     }
     downloadNextFile();
-  });
+  }).then(() => writeStateFile(packwiz, hashes));
 }