add dataPath option to allow working with arrays wrapped in objects.

raycast · Mar 18, 2024 · 76c9f3a · 76c9f3a
1 parent e683e1b
commit 76c9f3a
Show file tree

Hide file tree

Showing 4 changed files with 215 additions and 37 deletions.
diff --git a/docs/utils-reference/react-hooks/useStreamJSON.md b/docs/utils-reference/react-hooks/useStreamJSON.md
@@ -35,6 +35,7 @@ With a few options:
 - `options.fileName` is the name of the file where the JSON content will be cached. By default, `cache.json` will be used.
 - `options.folder` the folder where to cache the JSON. By default, `environment.supportPath` will be used.
 - `options.pageSize` the amount of items to fetch at a time. By default, 20 will be used
+- `options.dataPath` is a string informing the hook that the array of data is wrapped inside one or multiple objects, and the path it needs to take to get to it.
 - `options.transform` is a function called with each object encountered while streaming. The result of this function is what will be passed to `options.filter`. Note that the hook will revalidate every time the filter function changes, so you need to use [useCallback](https://react.dev/reference/react/useCallback) to make sure it only changes when it needs to.
 - `options.filter` is a function called with each object encountered while streaming. If it returns `true`, the object will be kept, otherwise it will be discarded. Note that the hook will revalidate every time the filter function changes, so you need to use [useCallback](https://react.dev/reference/react/useCallback) to make sure it only changes when it needs to.
 

diff --git a/src/useStreamJSON.ts b/src/useStreamJSON.ts
@@ -7,10 +7,11 @@ import { pipeline } from "node:stream/promises";
 import { useRef } from "react";
 import Chain from "stream-chain";
 import { parser } from "stream-json";
+import Pick from "stream-json/filters/Pick";
 import StreamArray from "stream-json/streamers/StreamArray";
 import { isJSON } from "./fetch-utils";
-import { CachedPromiseOptions, useCachedPromise } from "./useCachedPromise";
 import { FunctionReturningPaginatedPromise, UseCachedPromiseReturnType } from "./types";
+import { CachedPromiseOptions, useCachedPromise } from "./useCachedPromise";
 
 async function cache(url: RequestInfo, destination: string, fetchOptions?: RequestInit) {
   if (typeof url === "object" || url.startsWith("http://") || url.startsWith("https://")) {
@@ -106,25 +107,18 @@ async function* streamJsonFile<T>(
   filePath: string,
   pageSize: number,
   abortSignal?: AbortSignal,
+  dataPath?: string,
   filterFn?: (item: T) => boolean,
   transformFn?: (item: any) => T,
 ): AsyncGenerator<T[]> {
   let page: T[] = [];
-  const fileStream = createReadStream(filePath);
-  const jsonParser = parser();
-  const arrayParser = new StreamArray();
 
-  const pipeline = new Chain([fileStream, jsonParser, arrayParser]);
+  const pipeline = new Chain([
+    createReadStream(filePath),
+    dataPath ? Pick.withParser({ filter: dataPath }) : parser(),
+    new StreamArray(),
+  ]);
 
-  fileStream.on("error", (_error) => {
-    pipeline.destroy();
-  });
-  jsonParser.on("error", (_error) => {
-    pipeline.destroy();
-  });
-  arrayParser.on("error", (_error) => {
-    pipeline.destroy();
-  });
   abortSignal?.addEventListener("abort", () => {
     pipeline.destroy();
   });
@@ -144,13 +138,16 @@ async function* streamJsonFile<T>(
         page = [];
       }
     }
-    if (page.length > 0) {
-      yield page;
-    }
   } catch (e) {
     pipeline.destroy();
     return [];
   }
+
+  if (page.length > 0) {
+    yield page;
+  }
+
+  return [];
 }
 
 type Options<T> = {
@@ -166,6 +163,14 @@ type Options<T> = {
    * @remark If the folder doesn't exist, the hook will try to create it, and any intermediate folders.
    */
   folder?: string;
+  /**
+   * The hook expects to iterate through an array of data, so by default, it assumes the JSON it receives itself represents an array. However, sometimes the array of data is wrapped in an object,
+   * i.e. `{ "success": true, "data": […] }`, or even `{ "success": true, "results": { "data": […] } }`. In those cases, you can use `dataPath` to specify where the data array can be found.
+   *
+   * @example For `{ "success": true, "data": […] }`, dataPath would be `data`
+   * @example For `{ "success": true, "results": { "data": […] } }`, dataPath would be `results.data`
+   */
+  dataPath?: string;
   /**
    * A function to decide whether a particular item should be kept or not.
    * Defaults to `undefined`, keeping any encountered item.
@@ -353,6 +358,7 @@ export function useStreamJSON<T, U extends any[] = any[]>(
     onData,
     onWillExecute,
     fileName,
+    dataPath,
     filter,
     transform,
     folder = environment.supportPath,
@@ -380,8 +386,9 @@ export function useStreamJSON<T, U extends any[] = any[]>(
       folder: string,
       fileName: string,
       fetchOptions: RequestInit | undefined,
+      dataPath: string | undefined,
       filter: ((item: T) => boolean) | undefined,
-      transform: ((item: any) => T) | undefined,
+      transform: ((item: unknown) => T) | undefined,
     ) =>
       async ({ page }) => {
         if (page === 0) {
@@ -393,6 +400,7 @@ export function useStreamJSON<T, U extends any[] = any[]>(
             destination,
             pageSize,
             controllerRef.current?.signal,
+            dataPath,
             filter,
             transform,
           );
@@ -404,7 +412,16 @@ export function useStreamJSON<T, U extends any[] = any[]>(
         hasMoreRef.current = !done;
         return { hasMore: hasMoreRef.current, data: (newData ?? []) as T[] };
       },
-    [url, pageSize, folder, `${fileName?.replace(/\.json$/, "") ?? "cache"}.json`, fetchOptions, filter, transform],
+    [
+      url,
+      pageSize,
+      folder,
+      `${fileName?.replace(/\.json$/, "") ?? "cache"}.json`,
+      fetchOptions,
+      dataPath,
+      filter,
+      transform,
+    ],
     useCachedPromiseOptions,
   );
 }
diff --git a/tests/assets/stream-json-nested-object.json b/tests/assets/stream-json-nested-object.json
@@ -0,0 +1,104 @@
+{
+  "nested": {
+    "data": [
+      "foo1Lorem",
+      "bar2Ipsum",
+      "baz3Dolor",
+      "sit4Amet",
+      "con5Secte",
+      "tur6Adipi",
+      "sci7Elit",
+      "sed8DoEiu",
+      "mod9Tempo",
+      "inc10Idunt",
+      "ut11Labo",
+      "et12Dolo",
+      "magn13Alia",
+      "no14Sea",
+      "tak15Imata",
+      "san16ctus",
+      "est17Lore",
+      "dol18Lorem",
+      "ips19Amet",
+      "sit20Conse",
+      "ctet21ur",
+      "adip22Sci",
+      "elit23Sed",
+      "eius24Mod",
+      "temp25Orin",
+      "cid26Untut",
+      "lab27Oret",
+      "dolo28remag",
+      "na29Aliqu",
+      "no30Num",
+      "quam31Erat",
+      "volu32Patat",
+      "vel33Illu",
+      "fac34Pris",
+      "at35Vero",
+      "eos36Etac",
+      "cus37AmNon",
+      "proi38Dent",
+      "sunt39In",
+      "culp40Itqui",
+      "offic41Iades",
+      "dolor42Mag",
+      "fug43IatNull",
+      "pari44aturXcep",
+      "teur45Sint",
+      "occa46CatCup",
+      "idat47Non",
+      "pro48Iden",
+      "sun49Incu",
+      "culp50Atqu",
+      "offi51Ader",
+      "moll52Itan",
+      "anim53Ides",
+      "est54Labor",
+      "dolo55remi",
+      "min56Imve",
+      "quis57Nost",
+      "exer58Tation",
+      "ulla59Mco",
+      "labo60Risi",
+      "nisi61Utali",
+      "quip62Exea",
+      "comm63Odo",
+      "cons64Equa",
+      "tqui67Offi",
+      "cian68Denon",
+      "labor69Sum",
+      "nisi70Utal",
+      "quip71Exce",
+      "comm72Odio",
+      "conse73Quat",
+      "duis74Aute",
+      "irur75Dolor",
+      "in76Repre",
+      "hend77Volup",
+      "veli78Esse",
+      "cilum79Dolore",
+      "eu80Fugiat",
+      "nulla81Pari",
+      "except82Sint",
+      "occaec83Catcupi",
+      "non84Proident",
+      "sunt85Inculpa",
+      "quip86Excea",
+      "commo87Dios",
+      "conse88Sequat",
+      "duis89Autei",
+      "urure90Dolo",
+      "repre91Hende",
+      "volup92Velit",
+      "esse93Cillum",
+      "eu94Fugia",
+      "nulla95Paria",
+      "excepteur96Sint",
+      "occaecat97Cupidat",
+      "non98Proiden",
+      "sunt99Inculp",
+      "foo100BarBaz"
+    ]
+  }
+}
diff --git a/tests/src/stream-json-paginated.tsx b/tests/src/stream-json-paginated.tsx
@@ -10,7 +10,7 @@ type Cask = { token: string; name: string[]; desc?: string };
 
 export default function Main(): JSX.Element {
   const [searchText, setSearchText] = useState("");
-  const [type, setType] = useCachedState<"cask" | "formula">("cask");
+  const [type, setType] = useCachedState<"cask" | "formula" | "nestedData">("cask");
 
   const formulaFilter = useCallback(
     (item: Formula) => {
@@ -24,18 +24,6 @@ export default function Main(): JSX.Element {
     return { name: item.name, desc: item.desc };
   }, []);
 
-  const caskFilter = useCallback(
-    (item: Cask) => {
-      if (!searchText) return true;
-      return item.name.join(",").toLocaleLowerCase().includes(searchText);
-    },
-    [searchText],
-  );
-
-  const caskTransform = useCallback((item: any): Cask => {
-    return { token: item.token, name: item.name, desc: item.desc };
-  }, []);
-
   const {
     data: formulae,
     mutate: mutateFormulae,
@@ -51,6 +39,18 @@ export default function Main(): JSX.Element {
     execute: type === "formula",
   });
 
+  const caskFilter = useCallback(
+    (item: Cask) => {
+      if (!searchText) return true;
+      return item.name.join(",").toLocaleLowerCase().includes(searchText);
+    },
+    [searchText],
+  );
+
+  const caskTransform = useCallback((item: any): Cask => {
+    return { token: item.token, name: item.name, desc: item.desc };
+  }, []);
+
   const {
     data: casks,
     mutate: mutateCasks,
@@ -66,25 +66,52 @@ export default function Main(): JSX.Element {
     execute: type === "cask",
   });
 
+  const nestedDataFilter = useCallback(
+    (item: string) => {
+      if (!searchText) return true;
+      return item.toLocaleLowerCase().includes(searchText);
+    },
+    [searchText],
+  );
+
+  const nestedDataTransform = useCallback((item: string): string => {
+    return item.toLocaleLowerCase();
+  }, []);
+
+  const {
+    data: nestedData,
+    mutate: mutateNestedData,
+    isLoading: isLoadingDataKey,
+    pagination: nestedDataPagination,
+  } = useStreamJSON(`file:///${join(environment.assetsPath, "stream-json-nested-object.json")}`, {
+    initialData: [] as string[],
+    fileName: "nested-data-cache",
+    dataPath: "nested.data",
+    filter: nestedDataFilter,
+    transform: nestedDataTransform,
+    execute: type === "nestedData",
+  });
+
   return (
     <List
-      isLoading={isLoadingFormulae || isLoadingCasks}
-      pagination={type === "cask" ? caskPagination : formulaPagination}
+      isLoading={isLoadingFormulae || isLoadingCasks || isLoadingDataKey}
+      pagination={type === "cask" ? caskPagination : type === "formula" ? formulaPagination : nestedDataPagination}
       onSearchTextChange={setSearchText}
       searchBarAccessory={
         <List.Dropdown
           value={type}
           tooltip=""
           onChange={(newValue: string) => {
-            setType(newValue as "cask" | "formula");
+            setType(newValue as "cask" | "formula" | "nestedData");
           }}
         >
           <List.Dropdown.Item title="Casks" value="cask" />
           <List.Dropdown.Item title="Formulae" value="formula" />
+          <List.Dropdown.Item title="Nested Data" value="nestedData" />
         </List.Dropdown>
       }
     >
-      {type === "cask" ? (
+      {type === "cask" && (
         <List.Section title="Casks">
           {casks?.map((d) => {
             return (
@@ -110,7 +137,9 @@ export default function Main(): JSX.Element {
             );
           })}
         </List.Section>
-      ) : (
+      )}
+
+      {type === "formula" && (
         <List.Section title="Formulae">
           {formulae?.map((d) => {
             return (
@@ -137,6 +166,33 @@ export default function Main(): JSX.Element {
           })}
         </List.Section>
       )}
+
+      {type === "nestedData" && (
+        <List.Section title="Nested Data">
+          {nestedData?.map((d) => {
+            return (
+              <List.Item
+                key={d}
+                title={d}
+                actions={
+                  <ActionPanel>
+                    <Action
+                      title="Delete All Items But This One"
+                      onAction={async () => {
+                        mutateNestedData(setTimeout(5000), {
+                          optimisticUpdate: () => {
+                            return [d];
+                          },
+                        });
+                      }}
+                    />
+                  </ActionPanel>
+                }
+              />
+            );
+          })}
+        </List.Section>
+      )}
     </List>
   );
 }