diff --git a/docs/utils-reference/getting-started.md b/docs/utils-reference/getting-started.md index 94bbd03..cba8f29 100644 --- a/docs/utils-reference/getting-started.md +++ b/docs/utils-reference/getting-started.md @@ -16,6 +16,10 @@ npm install --save @raycast/utils ## Changelog +### v1.14.0 + +- Add `useStreamJSON` hook. + ### v1.13.6 - Updated `useFetch`'s `mapResult` type to allow returning `cursor` in addition to `data` and `hasMore`. diff --git a/docs/utils-reference/react-hooks/useCachedPromise.md b/docs/utils-reference/react-hooks/useCachedPromise.md index d8d4ed1..934d6cd 100644 --- a/docs/utils-reference/react-hooks/useCachedPromise.md +++ b/docs/utils-reference/react-hooks/useCachedPromise.md @@ -26,8 +26,8 @@ function useCachedPromise( execute?: boolean; onError?: (error: Error) => void; onData?: (data: Result) => void; - onWillExecute?: (args: Parameters) -> void; - } + onWillExecute?: (args: Parameters) => void; + }, ): AsyncState> & { revalidate: () => void; mutate: MutatePromise | U>; diff --git a/docs/utils-reference/react-hooks/useFetch.md b/docs/utils-reference/react-hooks/useFetch.md index 5a26100..5b15407 100644 --- a/docs/utils-reference/react-hooks/useFetch.md +++ b/docs/utils-reference/react-hooks/useFetch.md @@ -13,14 +13,14 @@ export function useFetch( url: RequestInfo, options?: RequestInit & { parseResponse?: (response: Response) => Promise; - mapResult?: (result: V) => { data: T } + mapResult?: (result: V) => { data: T }; initialData?: U; keepPreviousData?: boolean; execute?: boolean; onError?: (error: Error) => void; onData?: (data: T) => void; - onWillExecute?: (args: [string, RequestInit]) -> void; - } + onWillExecute?: (args: [string, RequestInit]) => void; + }, ): AsyncState & { revalidate: () => void; mutate: MutatePromise; diff --git a/docs/utils-reference/react-hooks/usePromise.md b/docs/utils-reference/react-hooks/usePromise.md index 49033eb..e13b9fb 100644 --- a/docs/utils-reference/react-hooks/usePromise.md +++ b/docs/utils-reference/react-hooks/usePromise.md @@ -19,8 +19,8 @@ function usePromise( execute?: boolean; onError?: (error: Error) => void; onData?: (data: Result) => void; - onWillExecute?: (args: Parameters) -> void; - } + onWillExecute?: (args: Parameters) => void; + }, ): AsyncState> & { revalidate: () => void; mutate: MutatePromise | undefined>; diff --git a/docs/utils-reference/react-hooks/useStreamJSON.md b/docs/utils-reference/react-hooks/useStreamJSON.md new file mode 100644 index 0000000..7d57b7c --- /dev/null +++ b/docs/utils-reference/react-hooks/useStreamJSON.md @@ -0,0 +1,209 @@ +# `useStreamJSON` + +Hook which takes a `http://`, `https://` or `file:///` URL pointing to a JSON resource, caches it to the command's support folder, and streams through its content. Useful when dealing with large JSON arrays which would be too big to fit in the command's memory. + +## Signature + +```ts +export function useStreamJSON( + url: RequestInfo, + options: RequestInit & { + filter?: (item: T) => boolean; + transform?: (item: any) => T; + pageSize?: number; + initialData?: U; + keepPreviousData?: boolean; + execute?: boolean; + onError?: (error: Error) => void; + onData?: (data: T) => void; + onWillExecute?: (args: [string, RequestInit]) => void; + }, +): AsyncState> & { + revalidate: () => void; +}; +``` + +### Arguments + +- `url` - The [`RequestInfo`](https://github.com/nodejs/undici/blob/v5.7.0/types/fetch.d.ts#L12) describing the resource that needs to be fetched. Strings starting with `http://`, `https://` and `Request` objects will use `fetch`, while strings starting with `file:///` will be copied to the cache folder. + +With a few options: + +- `options` extends [`RequestInit`](https://github.com/nodejs/undici/blob/v5.7.0/types/fetch.d.ts#L103-L117) allowing you to specify a body, headers, etc. to apply to the request. +- `options.pageSize` the amount of items to fetch at a time. By default, 20 will be used +- `options.dataPath` is a string or regular expression informing the hook that the array (or arrays) of data you want to stream through is wrapped inside one or multiple objects, and it indicates the path it needs to take to get to it. +- `options.transform` is a function called with each top-level object encountered while streaming. If the function returns an array, the hook will end up streaming through its children, and each array item will be passed to `options.filter`. If the function returns something other than an array, _it_ will be passed to `options.filter`. Note that the hook will revalidate every time the filter function changes, so you need to use [useCallback](https://react.dev/reference/react/useCallback) to make sure it only changes when it needs to. +- `options.filter` is a function called with each object encountered while streaming. If it returns `true`, the object will be kept, otherwise it will be discarded. Note that the hook will revalidate every time the filter function changes, so you need to use [useCallback](https://react.dev/reference/react/useCallback) to make sure it only changes when it needs to. + +Including the [useCachedPromise](./useCachedPromise.md)'s options: + +- `options.keepPreviousData` is a boolean to tell the hook to keep the previous results instead of returning the initial value if there aren't any in the cache for the new arguments. This is particularly useful when used for data for a List to avoid flickering. + +Including the [useCachedState](./useCachedState.md)'s options: + +- `options.initialData` is the initial value of the state if there aren't any in the Cache yet. + +Including the [usePromise](./usePromise.md)'s options: + +- `options.execute` is a boolean to indicate whether to actually execute the function or not. This is useful for cases where one of the function's arguments depends on something that might not be available right away (for example, depends on some user inputs). Because React requires every hook to be defined on the render, this flag enables you to define the hook right away but wait until you have all the arguments ready to execute the function. +- `options.onError` is a function called when an execution fails. By default, it will log the error and show a generic failure toast with an action to retry. +- `options.onData` is a function called when an execution succeeds. +- `options.onWillExecute` is a function called when an execution will start.. + +### Return + +Returns an object with the [AsyncState](#asyncstate) corresponding to the execution of the fetch as well as a couple of methods to manipulate it. + +- `data`, `error`, `isLoading` - see [AsyncState](#asyncstate). +- `pagination` - the pagination object that Raycast [`List`s](https://developers.raycast.com/api-reference/user-interface/list#props) and [`Grid`s](https://developers.raycast.com/api-reference/user-interface/grid#props) expect. +- `revalidate` is a method to manually call the function with the same arguments again. +- `mutate` is a method to wrap an asynchronous update and gives some control over how the hook's data should be updated while the update is going through. By default, the data will be revalidated (eg. the function will be called again) after the update is done. See [Mutation and Optimistic Updates](#mutation-and-optimistic-updates) for more information. + +## Example + +```ts +import { Action, ActionPanel, List, environment } from "@raycast/api"; +import { useStreamJSON } from "@raycast/utils"; +import { join } from "path"; +import { useCallback, useState } from "react"; + +type Formula = { name: string; desc?: string }; + +export default function Main(): JSX.Element { + const [searchText, setSearchText] = useState(""); + + const formulaFilter = useCallback( + (item: Formula) => { + if (!searchText) return true; + return item.name.toLocaleLowerCase().includes(searchText); + }, + [searchText], + ); + + const formulaTransform = useCallback((item: any): Formula => { + return { name: item.name, desc: item.desc }; + }, []); + + const { data, isLoading, pagination } = useStreamJSON("https://formulae.brew.sh/api/formula.json", { + initialData: [] as Formula[], + pageSize: 20, + filter: formulaFilter, + transform: formulaTransform + }); + + return ( + + + {data.map((d) => ( + + ))} + + + ); +} +``` + +## Mutation and Optimistic Updates + +In an optimistic update, the UI behaves as though a change was successfully completed before receiving confirmation from the server that it was - it is being optimistic that it will eventually get the confirmation rather than an error. This allows for a more responsive user experience. + +You can specify an `optimisticUpdate` function to mutate the data in order to reflect the change introduced by the asynchronous update. + +When doing so, you can specify a `rollbackOnError` function to mutate back the data if the asynchronous update fails. If not specified, the data will be automatically rolled back to its previous value (before the optimistic update). + +```tsx +import { Action, ActionPanel, List, environment } from "@raycast/api"; +import { useStreamJSON } from "@raycast/utils"; +import { join } from "path"; +import { useCallback, useState } from "react"; +import { setTimeout } from "timers/promises"; + +type Formula = { name: string; desc?: string }; + +export default function Main(): JSX.Element { + const [searchText, setSearchText] = useState(""); + + const formulaFilter = useCallback( + (item: Formula) => { + if (!searchText) return true; + return item.name.toLocaleLowerCase().includes(searchText); + }, + [searchText], + ); + + const formulaTransform = useCallback((item: any): Formula => { + return { name: item.name, desc: item.desc }; + }, []); + + const { data, isLoading, mutate, pagination } = useStreamJSON("https://formulae.brew.sh/api/formula.json", { + initialData: [] as Formula[], + pageSize: 20, + filter: formulaFilter, + transform: formulaTransform, + }); + + return ( + + + {data.map((d) => ( + + { + mutate(setTimeout(1000), { + optimisticUpdate: () => { + return [d]; + }, + }); + }} + /> + + } + /> + ))} + + + ); +} +``` + +## Types + +### AsyncState + +An object corresponding to the execution state of the function. + +```ts +// Initial State +{ + isLoading: true, // or `false` if `options.execute` is `false` + data: undefined, + error: undefined +} + +// Success State +{ + isLoading: false, + data: T, + error: undefined +} + +// Error State +{ + isLoading: false, + data: undefined, + error: Error +} + +// Reloading State +{ + isLoading: true, + data: T | undefined, + error: Error | undefined +} +``` diff --git a/package-lock.json b/package-lock.json index 6d8ca8d..d7a2487 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@raycast/utils", - "version": "1.13.6", + "version": "1.14.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@raycast/utils", - "version": "1.13.6", + "version": "1.14.0", "license": "MIT", "dependencies": { "content-type": "^1.0.5", @@ -14,7 +14,9 @@ "dequal": "^2.0.3", "media-typer": "^1.1.0", "object-hash": "^3.0.0", - "signal-exit": "^4.0.2" + "signal-exit": "^4.0.2", + "stream-chain": "^2.2.5", + "stream-json": "^1.8.0" }, "devDependencies": { "@raycast/api": "1.52.0", @@ -22,6 +24,8 @@ "@types/media-typer": "^1.1.1", "@types/object-hash": "^3.0.4", "@types/signal-exit": "^3.0.2", + "@types/stream-chain": "^2.0.4", + "@types/stream-json": "^1.7.7", "@typescript-eslint/eslint-plugin": "6.7.5", "@typescript-eslint/parser": "6.7.5", "eslint": "8.51.0", @@ -294,6 +298,25 @@ "integrity": "sha512-ofhmMb/rrf446sNpLoBb2UPyNQv07DEotKnlV+elPHkbS4Ykyp7WzKoLPpkBp2KclG0UV72pMLACQtn2anvdrg==", "dev": true }, + "node_modules/@types/stream-chain": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.4.tgz", + "integrity": "sha512-V7TsWLHrx79KumkHqSD7F8eR6POpEuWb6PuXJ7s/dRHAf3uVst3Jkp1yZ5XqIfECZLQ4a28vBVstTErmsMBvaQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/stream-json": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.7.tgz", + "integrity": "sha512-hHG7cLQ09H/m9i0jzL6UJAeLLxIWej90ECn0svO4T8J0nGcl89xZDQ2ujT4WKlvg0GWkcxJbjIDzW/v7BYUM6Q==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.7.5", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", @@ -3447,6 +3470,19 @@ "node": ">=8" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==" + }, + "node_modules/stream-json": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.8.0.tgz", + "integrity": "sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", @@ -4086,6 +4122,25 @@ "integrity": "sha512-ofhmMb/rrf446sNpLoBb2UPyNQv07DEotKnlV+elPHkbS4Ykyp7WzKoLPpkBp2KclG0UV72pMLACQtn2anvdrg==", "dev": true }, + "@types/stream-chain": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.0.4.tgz", + "integrity": "sha512-V7TsWLHrx79KumkHqSD7F8eR6POpEuWb6PuXJ7s/dRHAf3uVst3Jkp1yZ5XqIfECZLQ4a28vBVstTErmsMBvaQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/stream-json": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.7.tgz", + "integrity": "sha512-hHG7cLQ09H/m9i0jzL6UJAeLLxIWej90ECn0svO4T8J0nGcl89xZDQ2ujT4WKlvg0GWkcxJbjIDzW/v7BYUM6Q==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/stream-chain": "*" + } + }, "@typescript-eslint/eslint-plugin": { "version": "6.7.5", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", @@ -6231,6 +6286,19 @@ "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, + "stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==" + }, + "stream-json": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.8.0.tgz", + "integrity": "sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw==", + "requires": { + "stream-chain": "^2.2.5" + } + }, "string.prototype.trim": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", diff --git a/package.json b/package.json index 8eafd89..95f3e76 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@raycast/utils", - "version": "1.13.6", + "version": "1.14.0", "description": "Set of utilities to streamline building Raycast extensions", "author": "Raycast Technologies Ltd.", "homepage": "https://developers.raycast.com/utils-reference", @@ -33,7 +33,9 @@ "dequal": "^2.0.3", "media-typer": "^1.1.0", "object-hash": "^3.0.0", - "signal-exit": "^4.0.2" + "signal-exit": "^4.0.2", + "stream-chain": "^2.2.5", + "stream-json": "^1.8.0" }, "devDependencies": { "@raycast/api": "1.52.0", @@ -41,6 +43,8 @@ "@types/media-typer": "^1.1.1", "@types/object-hash": "^3.0.4", "@types/signal-exit": "^3.0.2", + "@types/stream-chain": "^2.0.4", + "@types/stream-json": "^1.7.7", "@typescript-eslint/eslint-plugin": "6.7.5", "@typescript-eslint/parser": "6.7.5", "eslint": "8.51.0", diff --git a/src/fetch-utils.ts b/src/fetch-utils.ts new file mode 100644 index 0000000..70e3c11 --- /dev/null +++ b/src/fetch-utils.ts @@ -0,0 +1,27 @@ +import mediaTyper from "media-typer"; +import contentType from "content-type"; + +export function isJSON(contentTypeHeader: string | null | undefined): boolean { + if (contentTypeHeader) { + const ct = contentType.parse(contentTypeHeader); + + const mediaType = mediaTyper.parse(ct.type); + + if (mediaType.subtype === "json") { + return true; + } + + if (mediaType.suffix === "json") { + return true; + } + + if (mediaType.suffix && /\bjson\b/i.test(mediaType.suffix)) { + return true; + } + + if (mediaType.subtype && /\bjson\b/i.test(mediaType.subtype)) { + return true; + } + } + return false; +} diff --git a/src/index.ts b/src/index.ts index ebeb14c..a031a03 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ export * from "./useCachedState"; export * from "./useCachedPromise"; export * from "./useFetch"; export * from "./useExec"; +export * from "./useStreamJSON"; export * from "./useSQL"; export * from "./useForm"; export * from "./useAI"; diff --git a/src/useFetch.ts b/src/useFetch.ts index abfb6a1..b2a3a44 100644 --- a/src/useFetch.ts +++ b/src/useFetch.ts @@ -1,36 +1,10 @@ import { useCallback, useMemo, useRef } from "react"; import hash from "object-hash"; -import mediaTyper from "media-typer"; -import contentType from "content-type"; import { useCachedPromise, CachedPromiseOptions } from "./useCachedPromise"; import { useLatest } from "./useLatest"; import { FunctionReturningPaginatedPromise, FunctionReturningPromise, UseCachedPromiseReturnType } from "./types"; import { fetch } from "cross-fetch"; - -function isJSON(contentTypeHeader: string | null | undefined): boolean { - if (contentTypeHeader) { - const ct = contentType.parse(contentTypeHeader); - - const mediaType = mediaTyper.parse(ct.type); - - if (mediaType.subtype === "json") { - return true; - } - - if (mediaType.suffix === "json") { - return true; - } - - if (mediaType.suffix && /\bjson\b/i.test(mediaType.suffix)) { - return true; - } - - if (mediaType.subtype && /\bjson\b/i.test(mediaType.subtype)) { - return true; - } - } - return false; -} +import { isJSON } from "./fetch-utils"; async function defaultParsing(response: Response) { if (!response.ok) { diff --git a/src/useStreamJSON.ts b/src/useStreamJSON.ts new file mode 100644 index 0000000..7460593 --- /dev/null +++ b/src/useStreamJSON.ts @@ -0,0 +1,466 @@ +import { environment } from "@raycast/api"; +import fetch from "cross-fetch"; +import { createReadStream, createWriteStream, mkdirSync, Stats } from "node:fs"; +import { stat } from "node:fs/promises"; +import { join, normalize } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { useRef } from "react"; +import Chain from "stream-chain"; +import { parser } from "stream-json"; +import Pick from "stream-json/filters/Pick"; +import StreamArray from "stream-json/streamers/StreamArray"; +import { isJSON } from "./fetch-utils"; +import { Flatten, FunctionReturningPaginatedPromise, UseCachedPromiseReturnType } from "./types"; +import { CachedPromiseOptions, useCachedPromise } from "./useCachedPromise"; +import objectHash from "object-hash"; + +async function cache(url: RequestInfo, destination: string, fetchOptions?: RequestInit) { + if (typeof url === "object" || url.startsWith("http://") || url.startsWith("https://")) { + return await cacheURL(url, destination, fetchOptions); + } else if (url.startsWith("file://")) { + return await cacheFile( + normalize(decodeURIComponent(new URL(url).pathname)), + destination, + fetchOptions?.signal ? fetchOptions.signal : undefined, + ); + } else { + throw new Error("Only HTTP(S) or file URLs are supported"); + } +} + +async function cacheURL(url: RequestInfo, destination: string, fetchOptions?: RequestInit) { + const response = await fetch(url, fetchOptions); + + if (!response.ok) { + throw new Error("Failed to fetch URL"); + } + + if (!isJSON(response.headers.get("content-type"))) { + throw new Error("URL does not return JSON"); + } + if (!response.body) { + throw new Error("Failed to retrieve expected JSON content: Response body is missing or inaccessible."); + } + await pipeline( + response.body as unknown as NodeJS.ReadableStream, + createWriteStream(destination), + fetchOptions?.signal ? { signal: fetchOptions.signal } : undefined, + ); +} + +async function cacheFile(source: string, destination: string, abortSignal?: AbortSignal) { + await pipeline( + createReadStream(source), + createWriteStream(destination), + abortSignal ? { signal: abortSignal } : undefined, + ); +} + +async function cacheURLIfNecessary( + url: RequestInfo, + folder: string, + fileName: string, + forceUpdate: boolean, + fetchOptions?: RequestInit, +) { + const destination = join(folder, fileName); + + try { + await stat(folder); + } catch (e) { + mkdirSync(folder, { recursive: true }); + await cache(url, destination, fetchOptions); + return; + } + if (forceUpdate) { + await cache(url, destination, fetchOptions); + return; + } + + let stats: Stats | undefined = undefined; + try { + stats = await stat(destination); + } catch (e) { + await cache(url, destination, fetchOptions); + return; + } + + if (typeof url === "object" || url.startsWith("http://") || url.startsWith("https://")) { + const headResponse = await fetch(url, { ...fetchOptions, method: "HEAD" }); + if (!headResponse.ok) { + throw new Error("Could not fetch URL"); + } + + if (!isJSON(headResponse.headers.get("content-type"))) { + throw new Error("URL does not return JSON"); + } + + const lastModified = Date.parse(headResponse.headers.get("last-modified") ?? ""); + if (stats.size === 0 || Number.isNaN(lastModified) || lastModified > stats.mtimeMs) { + await cache(url, destination, fetchOptions); + return; + } + } else if (url.startsWith("file://")) { + try { + const sourceStats = await stat(normalize(decodeURIComponent(new URL(url).pathname))); + if (sourceStats.mtimeMs > stats.mtimeMs) { + await cache(url, destination, fetchOptions); + } + } catch (e) { + throw new Error("Source file could not be read"); + } + } else { + throw new Error("Only HTTP(S) or file URLs are supported"); + } +} + +async function* streamJsonFile( + filePath: string, + pageSize: number, + abortSignal?: AbortSignal, + dataPath?: string | RegExp, + filterFn?: (item: Flatten) => boolean, + transformFn?: (item: any) => T, +): AsyncGenerator { + let page: T extends unknown[] ? T : T[] = [] as T extends unknown[] ? T : T[]; + + const pipeline = new Chain([ + createReadStream(filePath), + dataPath ? Pick.withParser({ filter: dataPath }) : parser(), + new StreamArray(), + (data) => transformFn?.(data.value) ?? data.value, + ]); + + abortSignal?.addEventListener("abort", () => { + pipeline.destroy(); + }); + + try { + for await (const data of pipeline) { + if (abortSignal?.aborted) { + return []; + } + if (!filterFn || filterFn(data)) { + page.push(data); + } + if (page.length >= pageSize) { + yield page; + page = [] as T extends unknown[] ? T : T[]; + } + } + } catch (e) { + pipeline.destroy(); + throw e; + } + + if (page.length > 0) { + yield page; + } + + return []; +} + +type Options = { + /** + * The hook expects to iterate through an array of data, so by default, it assumes the JSON it receives itself represents an array. However, sometimes the array of data is wrapped in an object, + * i.e. `{ "success": true, "data": […] }`, or even `{ "success": true, "results": { "data": […] } }`. In those cases, you can use `dataPath` to specify where the data array can be found. + * + * @remark If your JSON object has multiple arrays that you want to stream data from, you can pass a regular expression to stream through all of them. + * + * @example For `{ "success": true, "data": […] }`, dataPath would be `data` + * @example For `{ "success": true, "results": { "data": […] } }`, dataPath would be `results.data` + * @example For `{ "success": true, "results": { "first_list": […], "second_list": […], "third_list": […] } }`, dataPath would be `/^results\.(first_list|second_list|third_list)$ +/`. + */ + dataPath?: string | RegExp; + /** + * A function to decide whether a particular item should be kept or not. + * Defaults to `undefined`, keeping any encountered item. + * + * @remark The hook will revalidate every time the filter function changes, so you need to use [useCallback](https://react.dev/reference/react/useCallback) to make sure it only changes when it needs to. + */ + filter?: (item: Flatten) => boolean; + /** + * A function to apply to each item as it is encountered. Useful for a couple of things: + * 1. ensuring that all items have the expected properties, and, as on optimization, for getting rid of the properties that you don't care about. + * 2. when top-level objects actually represent nested data, which should be flattened. In this case, `transform` can return an array of items, and the hook will stream through each one of those items, + * passing them to `filter` etc. + * + * Defaults to a passthrough function if not provided. + * + * @remark The hook will revalidate every time the transform function changes, so it is important to use [useCallback](https://react.dev/reference/react/useCallback) to ensure it only changes when necessary to prevent unnecessary re-renders or computations. + * + * @example + * ``` + * // For data: `{ "data": [ { "type": "folder", "name": "item 1", "children": [ { "type": "item", "name": "item 2" }, { "type": "item", "name": "item 3" } ] }, { "type": "folder", "name": "item 4", children: [] } ] }` + * + * type Item = { + * type: "item"; + * name: string; + * }; + * + * type Folder = { + * type: "folder"; + * name: string; + * children: (Item | Folder)[]; + * }; + * + * function flatten(item: Item | Folder): { name: string }[] { + * const flattened: { name: string }[] = []; + * if (item.type === "folder") { + * flattened.push(...item.children.map(flatten).flat()); + * } + * if (item.type === "item") { + * flattened.push({ name: item.name }); + * } + * return flattened; + * } + * + * const transform = useCallback(flatten, []); + * const filter = useCallback((item: { name: string }) => { + * … + * }) + * ``` + */ + transform?: (item: any) => T; + /** + * The amount of items to return for each page. + * Defaults to `20`. + */ + pageSize?: number; +}; + +/** + * Takes a `http://`, `https://` or `file:///` URL pointing to a JSON resource, caches it to the command's support + * folder, and streams through its content. Useful when dealing with large JSON arrays which would be too big to fit + * in the command's memory. + * + * @remark The JSON resource needs to consist of an array of objects + * + * @example + * ``` + * import { List } from "@raycast/api"; + * import { useStreamJSON } from "@raycast/utils"; + * + * type Formula = { name: string; desc?: string }; + * + * export default function Main(): JSX.Element { + * const { data, isLoading, pagination } = useStreamJSON("https://formulae.brew.sh/api/formula.json"); + * + * return ( + * + * + * {data?.map((d) => )} + * + * + * ); + * } + * ``` + * + * @example + * ``` + * import { List } from "@raycast/api"; + * import { useStreamJSON } from "@raycast/utils"; + * import { homedir } from "os"; + * import { join } from "path"; + * + * type Formula = { name: string; desc?: string }; + * + * export default function Main(): JSX.Element { + * const { data, isLoading, pagination } = useStreamJSON(`file:///${join(homedir(), "Downloads", "formulae.json")}`); + * + * return ( + * + * + * {data?.map((d) => )} + * + * + * ); + * } + * ``` + */ +export function useStreamJSON(url: RequestInfo): UseCachedPromiseReturnType; + +/** + * Takes a `http://`, `https://` or `file:///` URL pointing to a JSON resource, caches it to the command's support + * folder, and streams through its content. Useful when dealing with large JSON arrays which would be too big to fit + * in the command's memory. + * + * @remark The JSON resource needs to consist of an array of objects + * + * @example + * ``` + * import { List, environment } from "@raycast/api"; + * import { useStreamJSON } from "@raycast/utils"; + * import { join } from 'path'; + * import { useCallback, useState } from "react"; + * + * type Formula = { name: string; desc?: string }; + * + * export default function Main(): JSX.Element { + * const [searchText, setSearchText] = useState(""); + * + * const formulaFilter = useCallback( + * (item: Formula) => { + * if (!searchText) return true; + * return item.name.toLocaleLowerCase().includes(searchText); + * }, + * [searchText], + * ); + * + * const formulaTransform = useCallback((item: any): Formula => { + * return { name: item.name, desc: item.desc }; + * }, []); + * + * const { data, isLoading, pagination } = useStreamJSON("https://formulae.brew.sh/api/formula.json", { + * initialData: [] as Formula[], + * pageSize: 20, + * filter: formulaFilter, + * transform: formulaTransform, + * }); + * + * return ( + * + * + * {data.map((d) => ( + * + * ))} + * + * + * ); + * } + * ``` support folder, and streams through its content. + * + * @example + * ``` + * import { List, environment } from "@raycast/api"; + * import { useStreamJSON } from "@raycast/utils"; + * import { join } from "path"; + * import { homedir } from "os"; + * import { useCallback, useState } from "react"; + * + * type Formula = { name: string; desc?: string }; + * + * export default function Main(): JSX.Element { + * const [searchText, setSearchText] = useState(""); + * + * const formulaFilter = useCallback( + * (item: Formula) => { + * if (!searchText) return true; + * return item.name.toLocaleLowerCase().includes(searchText); + * }, + * [searchText], + * ); + * + * const formulaTransform = useCallback((item: any): Formula => { + * return { name: item.name, desc: item.desc }; + * }, []); + * + * const { data, isLoading, pagination } = useStreamJSON(`file:///${join(homedir(), "Downloads", "formulae.json")}`, { + * initialData: [] as Formula[], + * pageSize: 20, + * filter: formulaFilter, + * transform: formulaTransform, + * }); + * + * return ( + * + * + * {data.map((d) => ( + * + * ))} + * + * + * ); + * } + * ``` + */ +export function useStreamJSON( + url: RequestInfo, + options: Options & RequestInit & Omit, "abortable">, +): UseCachedPromiseReturnType; + +export function useStreamJSON( + url: RequestInfo, + options?: Options & RequestInit & Omit, "abortable">, +): UseCachedPromiseReturnType { + const { + initialData, + execute, + keepPreviousData, + onError, + onData, + onWillExecute, + dataPath, + filter, + transform, + pageSize = 20, + ...fetchOptions + } = options ?? {}; + const previousUrl = useRef(); + const previousDestination = useRef(); + + const useCachedPromiseOptions: CachedPromiseOptions = { + initialData, + execute, + keepPreviousData, + onError, + onData, + onWillExecute, + }; + + const generatorRef = useRef | null>(null); + const controllerRef = useRef(null); + const hasMoreRef = useRef(false); + + return useCachedPromise( + ( + url: RequestInfo, + pageSize: number, + fetchOptions: RequestInit | undefined, + dataPath: string | RegExp | undefined, + filter: ((item: Flatten) => boolean) | undefined, + transform: ((item: unknown) => T) | undefined, + ) => + async ({ page }) => { + const fileName = objectHash(url) + ".json"; + const folder = environment.supportPath; + if (page === 0) { + controllerRef.current?.abort(); + controllerRef.current = new AbortController(); + const destination = join(folder, fileName); + /** + * Force update the cache when the URL changes but the cache destination does not. + */ + const forceCacheUpdate = Boolean( + previousUrl.current && + previousUrl.current !== url && + previousDestination.current && + previousDestination.current === destination, + ); + previousUrl.current = url; + previousDestination.current = destination; + await cacheURLIfNecessary(url, folder, fileName, forceCacheUpdate, { + ...fetchOptions, + signal: controllerRef.current?.signal, + }); + generatorRef.current = streamJsonFile( + destination, + pageSize, + controllerRef.current?.signal, + dataPath, + filter, + transform, + ); + } + if (!generatorRef.current) { + return { hasMore: hasMoreRef.current, data: [] as T extends unknown[] ? T : T[] }; + } + const { value: newData, done } = await generatorRef.current.next(); + hasMoreRef.current = !done; + return { hasMore: hasMoreRef.current, data: (newData ?? []) as T extends unknown[] ? T : T[] }; + }, + [url, pageSize, fetchOptions, dataPath, filter, transform], + useCachedPromiseOptions, + ); +} diff --git a/tests/assets/stream-json-nested-object.json b/tests/assets/stream-json-nested-object.json new file mode 100644 index 0000000..e3159dc --- /dev/null +++ b/tests/assets/stream-json-nested-object.json @@ -0,0 +1,104 @@ +{ + "nested": { + "data": [ + "foo1Lorem", + "bar2Ipsum", + "baz3Dolor", + "sit4Amet", + "con5Secte", + "tur6Adipi", + "sci7Elit", + "sed8DoEiu", + "mod9Tempo", + "inc10Idunt", + "ut11Labo", + "et12Dolo", + "magn13Alia", + "no14Sea", + "tak15Imata", + "san16ctus", + "est17Lore", + "dol18Lorem", + "ips19Amet", + "sit20Conse", + "ctet21ur", + "adip22Sci", + "elit23Sed", + "eius24Mod", + "temp25Orin", + "cid26Untut", + "lab27Oret", + "dolo28remag", + "na29Aliqu", + "no30Num", + "quam31Erat", + "volu32Patat", + "vel33Illu", + "fac34Pris", + "at35Vero", + "eos36Etac", + "cus37AmNon", + "proi38Dent", + "sunt39In", + "culp40Itqui", + "offic41Iades", + "dolor42Mag", + "fug43IatNull", + "pari44aturXcep", + "teur45Sint", + "occa46CatCup", + "idat47Non", + "pro48Iden", + "sun49Incu", + "culp50Atqu", + "offi51Ader", + "moll52Itan", + "anim53Ides", + "est54Labor", + "dolo55remi", + "min56Imve", + "quis57Nost", + "exer58Tation", + "ulla59Mco", + "labo60Risi", + "nisi61Utali", + "quip62Exea", + "comm63Odo", + "cons64Equa", + "tqui67Offi", + "cian68Denon", + "labor69Sum", + "nisi70Utal", + "quip71Exce", + "comm72Odio", + "conse73Quat", + "duis74Aute", + "irur75Dolor", + "in76Repre", + "hend77Volup", + "veli78Esse", + "cilum79Dolore", + "eu80Fugiat", + "nulla81Pari", + "except82Sint", + "occaec83Catcupi", + "non84Proident", + "sunt85Inculpa", + "quip86Excea", + "commo87Dios", + "conse88Sequat", + "duis89Autei", + "urure90Dolo", + "repre91Hende", + "volup92Velit", + "esse93Cillum", + "eu94Fugia", + "nulla95Paria", + "excepteur96Sint", + "occaecat97Cupidat", + "non98Proiden", + "sunt99Inculp", + "foo100BarBaz" + ] + } +} diff --git a/tests/package-lock.json b/tests/package-lock.json index 37446e1..7a52cff 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -21,7 +21,7 @@ }, "..": { "name": "@raycast/utils", - "version": "1.13.0", + "version": "1.13.6", "license": "MIT", "dependencies": { "content-type": "^1.0.5", @@ -29,7 +29,9 @@ "dequal": "^2.0.3", "media-typer": "^1.1.0", "object-hash": "^3.0.0", - "signal-exit": "^4.0.2" + "signal-exit": "^4.0.2", + "stream-chain": "^2.2.5", + "stream-json": "^1.8.0" }, "devDependencies": { "@raycast/api": "1.52.0", @@ -37,6 +39,8 @@ "@types/media-typer": "^1.1.1", "@types/object-hash": "^3.0.4", "@types/signal-exit": "^3.0.2", + "@types/stream-chain": "^2.0.4", + "@types/stream-json": "^1.7.7", "@typescript-eslint/eslint-plugin": "6.7.5", "@typescript-eslint/parser": "6.7.5", "eslint": "8.51.0", @@ -4014,6 +4018,8 @@ "@types/media-typer": "^1.1.1", "@types/object-hash": "^3.0.4", "@types/signal-exit": "^3.0.2", + "@types/stream-chain": "^2.0.4", + "@types/stream-json": "^1.7.7", "@typescript-eslint/eslint-plugin": "6.7.5", "@typescript-eslint/parser": "6.7.5", "content-type": "^1.0.5", @@ -4028,6 +4034,8 @@ "media-typer": "^1.1.0", "object-hash": "^3.0.0", "signal-exit": "^4.0.2", + "stream-chain": "^2.2.5", + "stream-json": "^1.8.0", "typescript": "5.2.2" } }, diff --git a/tests/package.json b/tests/package.json index 069767a..7accb7b 100644 --- a/tests/package.json +++ b/tests/package.json @@ -178,6 +178,13 @@ "required": true } ] + }, + { + "name": "stream-json-paginated", + "title": "useStreamJSON - paginated", + "subtitle": "Utils Smoke Tests", + "description": "Utils Smoke Tests", + "mode": "view" } ], "dependencies": { diff --git a/tests/src/stream-json-paginated.tsx b/tests/src/stream-json-paginated.tsx new file mode 100644 index 0000000..243edd7 --- /dev/null +++ b/tests/src/stream-json-paginated.tsx @@ -0,0 +1,193 @@ +import { Action, ActionPanel, List, environment } from "@raycast/api"; +import { useCachedState, useStreamJSON } from "@raycast/utils"; +import { join } from "path"; +import { useCallback, useState } from "react"; +import { setTimeout } from "timers/promises"; + +type Formula = { name: string; desc?: string }; + +type Cask = { token: string; name: string[]; desc?: string }; + +export default function Main(): JSX.Element { + const [searchText, setSearchText] = useState(""); + const [type, setType] = useCachedState<"cask" | "formula" | "nestedData">("cask"); + + const formulaFilter = useCallback( + (item: Formula) => { + if (!searchText) return true; + return item.name.toLocaleLowerCase().includes(searchText); + }, + [searchText], + ); + + const formulaTransform = useCallback((item: any): Formula => { + return { name: item.name, desc: item.desc }; + }, []); + + const { + data: formulae, + mutate: mutateFormulae, + isLoading: isLoadingFormulae, + pagination: formulaPagination, + } = useStreamJSON("https://formulae.brew.sh/api/formula.json", { + initialData: [] as Formula[], + pageSize: 20, + filter: formulaFilter, + transform: formulaTransform, + execute: type === "formula", + }); + + const caskFilter = useCallback( + (item: Cask) => { + if (!searchText) return true; + return item.name.join(",").toLocaleLowerCase().includes(searchText); + }, + [searchText], + ); + + const caskTransform = useCallback((item: any): Cask => { + return { token: item.token, name: item.name, desc: item.desc }; + }, []); + + const { + data: casks, + mutate: mutateCasks, + isLoading: isLoadingCasks, + pagination: caskPagination, + } = useStreamJSON("https://formulae.brew.sh/api/cask.json", { + initialData: [] as Cask[], + pageSize: 20, + filter: caskFilter, + transform: caskTransform, + execute: type === "cask", + }); + + const nestedDataFilter = useCallback( + (item: string) => { + if (!searchText) return true; + return item.toLocaleLowerCase().includes(searchText); + }, + [searchText], + ); + + const nestedDataTransform = useCallback((item: string): string => { + return item.toLocaleLowerCase(); + }, []); + + const { + data: nestedData, + mutate: mutateNestedData, + isLoading: isLoadingDataKey, + pagination: nestedDataPagination, + } = useStreamJSON(`file:///${join(environment.assetsPath, "stream-json-nested-object.json")}`, { + initialData: [] as string[], + dataPath: /^nested.data$/, + filter: nestedDataFilter, + transform: nestedDataTransform, + execute: type === "nestedData", + }); + + return ( + { + setType(newValue as "cask" | "formula" | "nestedData"); + }} + > + + + + + } + > + {type === "cask" && ( + + {casks?.map((d) => { + return ( + + { + mutateCasks(setTimeout(5000), { + optimisticUpdate: () => { + return [d]; + }, + }); + }} + /> + + } + /> + ); + })} + + )} + + {type === "formula" && ( + + {formulae?.map((d) => { + return ( + + { + mutateFormulae(setTimeout(5000), { + optimisticUpdate: () => { + return [d]; + }, + }); + }} + /> + + } + /> + ); + })} + + )} + + {type === "nestedData" && ( + + {nestedData?.map((d) => { + return ( + + { + mutateNestedData(setTimeout(5000), { + optimisticUpdate: () => { + return [d]; + }, + }); + }} + /> + + } + /> + ); + })} + + )} + + ); +}