Merge pull request #1345 from ebkr/chunky

Use new Thunderstore API to fetch community's package list

src/components/mixins/SplashMixin.vue

@@ -2,10 +2,8 @@
 import Vue from 'vue';
 import Component from 'vue-class-component';
 
-import ApiResponse from '../../model/api/ApiResponse';
 import R2Error from '../../model/errors/R2Error';
 import RequestItem from '../../model/requests/RequestItem';
-import ConnectionProvider from '../../providers/generic/connection/ConnectionProvider';
 
 @Component
 export default class SplashMixin extends Vue {
@@ -52,26 +50,26 @@ export default class SplashMixin extends Vue {
     // Get the list of Thunderstore mods from API or local cache.
     async getThunderstoreMods() {
         this.loadingText = 'Connecting to Thunderstore';
-        let response: ApiResponse|undefined = undefined;
+        let packageListChunks: {full_name: string}[][]|undefined = undefined;
 
         const showProgress = (progress: number) => {
             this.loadingText = 'Getting mod list from Thunderstore';
             this.getRequestItem('ThunderstoreDownload').setProgress(progress);
         };
 
         try {
-            response = await ConnectionProvider.instance.getPackages(this.$store.state.activeGame, showProgress, 3);
+            packageListChunks = await this.$store.dispatch('tsMods/fetchPackageListChunks', showProgress);
         } catch (e) {
             console.error('SplashMixin failed to fetch mod list from API.', e);
         } finally {
             this.getRequestItem('ThunderstoreDownload').setProgress(100);
         }
 
-        if (response) {
+        if (packageListChunks) {
             this.loadingText = 'Storing the mod list into local cache';
 
             try {
-                await this.$store.dispatch('tsMods/updatePersistentCache', response.data);
+                await this.$store.dispatch('tsMods/updatePersistentCache', packageListChunks);
             } catch (e) {
                 console.error('SplashMixin failed to cache mod list locally.', e);
             }
@@ -98,10 +96,9 @@ export default class SplashMixin extends Vue {
 
         // To proceed, the loading of the mod list should result in a non-empty list.
         // Empty list is allowed if that's actually what the API returned.
-        if (
-            isModListLoaded &&
-            (this.$store.state.tsMods.mods.length || (response && !response.data.length))
-        ) {
+        const modListHasMods = this.$store.state.tsMods.mods.length;
+        const apiReturnedEmptyList = packageListChunks && packageListChunks[0].length === 0;
+        if (isModListLoaded && (modListHasMods || apiReturnedEmptyList)) {
             await this.moveToNextScreen();
         } else {
             this.heroTitle = 'Failed to get the list of online mods';

src/components/mixins/UtilityMixin.vue

@@ -4,7 +4,6 @@ import Component from 'vue-class-component';
 
 import R2Error from '../../model/errors/R2Error';
 import CdnProvider from '../../providers/generic/connection/CdnProvider';
-import ConnectionProvider from '../../providers/generic/connection/ConnectionProvider';
 
 @Component
 export default class UtilityMixin extends Vue {
@@ -16,8 +15,8 @@ export default class UtilityMixin extends Vue {
     }
 
     async refreshThunderstoreModList() {
-        const response = await ConnectionProvider.instance.getPackages(this.$store.state.activeGame);
-        await this.$store.dispatch("tsMods/updatePersistentCache", response.data);
+        const packageListChunks = await this.$store.dispatch('tsMods/fetchPackageListChunks');
+        await this.$store.dispatch("tsMods/updatePersistentCache", packageListChunks);
         await this.$store.dispatch("tsMods/updateMods");
         await this.$store.dispatch("profile/tryLoadModListFromDisk");
         await this.$store.dispatch("tsMods/prewarmCache");

src/r2mm/connection/ConnectionProviderImpl.ts

@@ -37,6 +37,7 @@ export default class ConnectionProviderImpl extends ConnectionProvider {
         return this.cleanExclusions(response.data as string);
     }
 
+    // Deprecated since ConnectionProviderImpl.getPackages is deprecated.
     private async getPackagesFromRemote(game: Game, downloadProgressed?: DownloadProgressed) {
         const response = await makeLongRunningGetRequest(
             game.thunderstoreUrl,
@@ -90,6 +91,7 @@ export default class ConnectionProviderImpl extends ConnectionProvider {
     }
 
     /**
+     * Deprecated: use TsModsModule.fetchPackageListChunks instead.
      * Return packages for given game from Thunderstore API
      */
     public async getPackages(game: Game, downloadProgressed?: DownloadProgressed, retries = 0): Promise<ApiResponse> {

src/r2mm/manager/PackageDexieStore.ts

@@ -1,7 +1,6 @@
 import Dexie, { Table } from 'dexie';
 
 import ThunderstoreMod from '../../model/ThunderstoreMod';
-import * as ArrayUtils from '../../utils/ArrayUtils';
 
 interface DexieVersion {
     full_name: string;
@@ -56,17 +55,19 @@ class PackageDexieStore extends Dexie {
 const db = new PackageDexieStore();
 
 // TODO: user type guards to validate (part of) the data before operations?
-export async function updateFromApiResponse(community: string, packages: any[]) {
+export async function updateFromApiResponse(community: string, packageChunks: any[][]) {
+    let default_order = 0;
     const extra = {community, date_fetched: new Date()};
-    const newPackageChunks: DexiePackage[][] = ArrayUtils.chunk(
-        packages.map((pkg, i) => ({
+    const newPackageChunks: DexiePackage[][] = packageChunks.map((chunk) =>
+        chunk.map((pkg) => ({
             ...pkg,
             ...extra,
-            default_order: i
-        })),
-        5000
+            default_order: default_order++
+        }))
     );
 
+    // Since we need to do these operations in a single transaction we can't
+    // process the chunks one by one as they are downloaded.
     await db.transaction(
         'rw',
         db.packages,

src/store/modules/TsModsModule.ts

@@ -6,7 +6,10 @@ import ThunderstoreCombo from '../../model/ThunderstoreCombo';
 import ThunderstoreMod from '../../model/ThunderstoreMod';
 import ConnectionProvider from '../../providers/generic/connection/ConnectionProvider';
 import * as PackageDb from '../../r2mm/manager/PackageDexieStore';
+import { isEmptyArray, isStringArray } from '../../utils/ArrayUtils';
+import { retry } from '../../utils/Common';
 import { Deprecations } from '../../utils/Deprecations';
+import { fetchAndProcessBlobFile } from '../../utils/HttpUtils';
 
 interface CachedMod {
     tsMod: ThunderstoreMod | undefined;
@@ -23,6 +26,16 @@ interface State {
     modsLastUpdated?: Date;
 }
 
+type ProgressCallback = (progress: number) => void;
+type PackageListChunk = {full_name: string}[];
+type ChunkedPackageList = PackageListChunk[];
+
+function isPackageListChunk(value: unknown): value is PackageListChunk {
+    return Array.isArray(value) && (
+        !value.length || typeof value[0].full_name === "string"
+    );
+}
+
 /**
  * For dealing with mods listed in communities, i.e. available through
  * the Thunderstore API. Mods received from the API are stored in
@@ -156,14 +169,60 @@ export const TsModsModule = {
     },
 
     actions: <ActionTree<State, RootState>>{
+        async _fetchPackageListIndex({rootState}): Promise<string[]> {
+            const indexUrl = rootState.activeGame.thunderstoreUrl;
+            const chunkIndex: string[] = await retry(() => fetchAndProcessBlobFile(indexUrl));
+
+            if (!isStringArray(chunkIndex)) {
+                throw new Error('Received invalid chunk index from API');
+            }
+            if (isEmptyArray(chunkIndex)) {
+                throw new Error('Received empty chunk index from API');
+            }
+
+            return chunkIndex;
+        },
+
+        async fetchPackageListChunks(
+            {dispatch},
+            progressCallback?: ProgressCallback
+        ): Promise<ChunkedPackageList> {
+            const chunkIndex: string[] = await dispatch('_fetchPackageListIndex');
+
+            // Count index as a chunk for progress bar purposes.
+            const chunkCount = chunkIndex.length + 1;
+            let completed = 1;
+            const updateProgress = () => progressCallback && progressCallback((completed / chunkCount) * 100);
+            updateProgress();
+
+            // Download chunks serially to avoid slow connections timing
+            // out due to concurrent requests competing for the bandwidth.
+            const chunks = [];
+            for (const [i, chunkUrl] of chunkIndex.entries()) {
+                const chunk = await retry(() => fetchAndProcessBlobFile(chunkUrl))
+
+                if (chunkIndex.length > 1 && isEmptyArray(chunkIndex)) {
+                    throw new Error(`Chunk #${i} in multichunk response was empty`);
+                } else if (!isPackageListChunk(chunk)) {
+                    throw new Error(`Chunk #${i} was invalid format`);
+                }
+
+                chunks.push(chunk);
+                completed++;
+                updateProgress();
+            }
+
+            return chunks;
+        },
+
         async prewarmCache({getters, rootGetters}) {
             const profileMods: ManifestV2[] = rootGetters['profile/modList'];
             profileMods.forEach(getters['cachedMod']);
         },
 
         async updateExclusions(
             {commit},
-            progressCallback?: (progress: number) => void
+            progressCallback?: ProgressCallback
         ) {
             const exclusions = await ConnectionProvider.instance.getExclusions(progressCallback);
             commit('setExclusions', exclusions);
@@ -181,13 +240,15 @@ export const TsModsModule = {
         /*** Save a mod list received from the Thunderstore API to IndexedDB */
         async updatePersistentCache(
             {dispatch, rootState, state},
-            packages: {full_name: string}[]
+            packages: ChunkedPackageList
         ) {
             if (state.exclusions === undefined) {
                 await dispatch('updateExclusions');
             }
 
-            const filtered = packages.filter((pkg) => !state.exclusions!.includes(pkg.full_name));
+            const filtered = packages.map((chunk) => chunk.filter(
+                (pkg) => !state.exclusions!.includes(pkg.full_name)
+            ));
             const community = rootState.activeGame.internalFolderName;
             await PackageDb.updateFromApiResponse(community, filtered);
         }

src/utils/ArrayUtils.ts

@@ -13,3 +13,11 @@ export function chunk<T>(original: T[], chunkSize: number): T[][] {
 
     return result;
 }
+
+export function isEmptyArray(value: unknown): boolean {
+    return Array.isArray(value) && !value.length;
+}
+
+export function isStringArray(value: any): value is string[] {
+    return Array.isArray(value) && value.every((item) => typeof item === 'string');
+}

src/utils/Common.ts

@@ -26,4 +26,28 @@ export const getPropertyFromPath = (object: Mappable, path: string | string[]):
     return undefined;
 };
 
+export async function retry<T>(
+    fn: () => Promise<T>,
+    attempts: number = 3,
+    canRetry: () => boolean = () => true,
+    onError: (e: Error | unknown) => void = console.error
+): Promise<T> {
+    for (let currentAttempt = 1; currentAttempt <= attempts; currentAttempt++) {
+        if (!canRetry()) {
+            throw new Error("Retry interrupted");
+        }
+
+        try {
+            return await fn();
+        } catch (e) {
+            onError(e);
+
+            if (currentAttempt < attempts) {
+                await sleep(5000);
+            }
+        }
+    }
+    throw new Error(`Retry failed after ${attempts} attempts!`);
+}
+
 export const sleep = (ms: number): Promise<void> => new Promise((res) => setTimeout(res, ms));

src/utils/GzipUtils.ts

@@ -0,0 +1,13 @@
+import * as zlib from 'zlib';
+
+export function decompressArrayBuffer(compressed: ArrayBuffer, encoding = 'utf-8'): Promise<string> {
+    return new Promise((resolve, reject) => {
+        zlib.gunzip(compressed, (err, result) => {
+            if (err) {
+                return reject(err);
+            }
+
+            return resolve(result.toString(encoding));
+        });
+    });
+}

src/utils/HttpUtils.ts

@@ -1,6 +1,7 @@
-import axios from "axios";
+import axios, { AxiosRequestConfig } from "axios";
 
 import { DownloadProgressed } from "../providers/generic/connection/ConnectionProvider";
+import { decompressArrayBuffer } from "./GzipUtils";
 
 const newAbortSignal = (timeoutMs: number) => {
     const abortController = new AbortController();
@@ -31,6 +32,8 @@ export const getAxiosWithTimeouts = (responseTimeout = 5000, totalTimeout = 1000
 };
 
 interface LongRunningRequestOptions {
+    /** Values passed as is to Axios constructor */
+    axiosConfig?: AxiosRequestConfig;
     /** Custom function to be called when progress is made. */
     downloadProgressed?: DownloadProgressed;
     /**
@@ -64,6 +67,7 @@ export const makeLongRunningGetRequest = async (
     options: Partial<LongRunningRequestOptions> = {}
 ) => {
     const {
+        axiosConfig = {},
         downloadProgressed = () => null,
         initialTimeout = 30 * 1000,
         totalTimeout = 5 * 60 * 1000,
@@ -87,6 +91,7 @@ export const makeLongRunningGetRequest = async (
     }
 
     const instance = axios.create({
+        ...axiosConfig,
         onDownloadProgress,
         signal: abortController.signal,
     });
@@ -99,6 +104,18 @@ export const makeLongRunningGetRequest = async (
     }
 }
 
+/**
+ * Download blob files containing gzip compressed JSON strings and
+ * return them as objects. This is used for data that's shared by
+ * all users and can be cached heavily on CDN level.
+ */
+export const fetchAndProcessBlobFile = async (url: string) => {
+    const response = await makeLongRunningGetRequest(url, {axiosConfig: {responseType: 'arraybuffer'}});
+    const buffer = Buffer.from(response.data);
+    const jsonString = await decompressArrayBuffer(buffer);
+    return JSON.parse(jsonString);
+}
+
 export const isNetworkError = (responseOrError: unknown) =>
     responseOrError instanceof Error && responseOrError.message === "Network Error";