Convert sqlite to pglite

.github/CONTRIBUTING.md

@@ -119,7 +119,7 @@ When adding new features or fixing bugs, it's important to add test cases to cov
 
 ### Run tests against Postgres
 
-By default, the test suite runs against in-memory SQLite databases which mimic Ponder development environments. Unless you are specifically testing Postgres behavior, you don't need to run tests against Postgres locally and can instead rely on CI to catch any regressions.
+By default, the test suite runs against in-memory PGlite databases which mimic Ponder development environments. Unless you are specifically testing Postgres behavior, you don't need to run tests against Postgres locally and can instead rely on CI to catch any regressions.
 
 To run the test suite against Postgres, set the `DATABASE_URL` env var in `packages/core/.env.local`.
 

.github/workflows/windows.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        database: [Postgres, SQLite]
+        database: [Postgres, PGLite]
     steps:
       - name: Clone repository
         uses: actions/checkout@v4
@@ -48,4 +48,4 @@ jobs:
       - name: Test
         run: pnpm --filter create-ponder test
         env:
-          ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }}
+          ETHERSCAN_API_KEY: ${{ secrets.ETHERSCAN_API_KEY }}

packages/core/src/_test/e2e/erc20/ponder.config.ts

@@ -13,7 +13,7 @@ function getDatabase() {
     const connectionString = databaseUrl.toString();
     return { kind: "postgres", connectionString } as const;
   } else {
-    return { kind: "sqlite" } as const;
+    return { kind: "pglite" } as const;
   }
 }
 

packages/core/src/_test/e2e/factory/ponder.config.ts

@@ -13,7 +13,7 @@ function getDatabase() {
     const connectionString = databaseUrl.toString();
     return { kind: "postgres", connectionString } as const;
   } else {
-    return { kind: "sqlite" } as const;
+    return { kind: "pglite" } as const;
   }
 }
 

packages/core/src/_test/setup.ts

@@ -71,7 +71,7 @@ export function setupCommon(context: TestContext) {
  *
  * If `process.env.DATABASE_URL` is set, creates a new database and drops
  * it in the cleanup function. If it's not set, creates a temporary directory
- * for SQLite and removes it in the cleanup function.
+ * for PGlite and removes it in the cleanup function.
  *
  * ```ts
  * // Add this to any test suite that uses the database.
@@ -135,36 +135,30 @@ export async function setupDatabaseServices(
   cleanup: () => Promise<void>;
 }> {
   const config = { ...defaultDatabaseServiceSetup, ...overrides };
-  console.log("create");
   const database = await createDatabase({
     common: context.common,
     databaseConfig: context.databaseConfig,
     schema: config.schema,
   });
-  console.log("db setup");
 
   await database.setup(config);
 
-  console.log("migrate sync");
   await database.migrateSync().catch((err) => {
     console.log(err);
     throw err;
   });
 
-  console.log("create sync store");
   const syncStore = createSyncStore({
     common: context.common,
     db: database.qb.sync,
   });
 
-  console.log("create ro store");
   const readonlyStore = getReadonlyStore({
     schema: config.schema,
     db: database.qb.user,
     common: context.common,
   });
 
-  console.log("create indexing store");
   const indexingStore =
     config.indexing === "historical"
       ? getHistoricalStore({

packages/core/src/common/options.ts

@@ -99,7 +99,7 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => {
 
     databaseHeartbeatInterval: 10 * 1000,
     databaseHeartbeatTimeout: 25 * 1000,
-    // Half of the max query parameters for SQLite
+    // Half of the max query parameters for PGlite
     databaseMaxQueryParameters: 16_000,
     databaseMaxRowLimit: 1_000,
 

packages/core/src/database/index.ts

@@ -96,19 +96,23 @@ type PonderInternalSchema = {
   [tableName: string]: UserTable;
 };
 
+type PGliteDriver = {
+  internal: PGlite;
+  readonly: PGlite;
+  user: PGlite;
+  sync: PGlite;
+};
+
+type PostgresDriver = {
+  internal: Pool;
+  user: Pool;
+  readonly: Pool;
+  sync: Pool;
+};
+
 type Driver<dialect extends "pglite" | "postgres"> = dialect extends "pglite"
-  ? {
-      internal: PGlite;
-      readonly: PGlite;
-      user: PGlite;
-      sync: PGlite;
-    }
-  : {
-      internal: Pool;
-      user: Pool;
-      readonly: Pool;
-      sync: Pool;
-    };
+  ? PGliteDriver
+  : PostgresDriver;
 
 type QueryBuilder = {
   /** For updating metadata and handling reorgs */
@@ -153,14 +157,11 @@ export const createDatabase = async (args: {
     const userDir = path.join(args.databaseConfig.directory, "public");
     const syncDir = path.join(args.databaseConfig.directory, "sync");
 
-    console.log("kysely create");
     const { client: userClient, dialect: userDialect } =
       await KyselyPGlite.create(userDir);
     const { client: syncClient, dialect: syncDialect } =
       await KyselyPGlite.create(syncDir);
 
-    console.log("after kysely create");
-
     driver = {
       internal: userClient,
       readonly: userClient,
@@ -216,6 +217,7 @@ export const createDatabase = async (args: {
             });
           }
         },
+        plugins: [new WithSchemaPlugin("ponder_sync")],
       }),
     };
   } else {
@@ -469,9 +471,7 @@ export const createDatabase = async (args: {
           migrationTableSchema: "ponder_sync",
         });
 
-        console.log("migrate to latest");
         const { error } = await migrator.migrateToLatest();
-        console.log(error);
         if (error) throw error;
       });
     },
@@ -1039,21 +1039,15 @@ export const createDatabase = async (args: {
       await qb.sync.destroy();
 
       if (dialect === "pglite") {
-        // @ts-ignore
-        driver.user.close();
-        // @ts-ignore
-        driver.readonly.close();
-        // @ts-ignore
-        driver.sync.close();
+        const d = driver as PGliteDriver;
+        await d.user.close();
+        await d.sync.close();
       } else {
-        // @ts-ignore
-        await driver.internal.end();
-        // @ts-ignore
-        await driver.user.end();
-        // @ts-ignore
-        await driver.readonly.end();
-        // @ts-ignore
-        await driver.sync.end();
+        const d = driver as PostgresDriver;
+        await d.internal.end();
+        await d.user.end();
+        await d.readonly.end();
+        await d.sync.end();
       }
 
       args.common.logger.debug({

packages/core/src/indexing-store/metadata.test.ts

@@ -13,19 +13,15 @@ beforeEach(setupIsolatedDatabase);
 const schema = createSchema(() => ({}));
 
 test("getMetadata() empty", async (context) => {
-  console.log("setup");
   const { database, cleanup } = await setupDatabaseServices(context, {
     schema,
   });
-  console.log("get store");
   const metadataStore = getMetadataStore({ db: database.qb.user });
 
-  console.log("get status");
   const status = await metadataStore.getStatus();
 
   expect(status).toBe(null);
 
-  console.log("cleanup");
   await cleanup();
 });