fix: create tpw table if doesn't exist

.github/workflows/ci.yml

@@ -11,12 +11,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Use Node.js 16.x
+      - name: Use Node.js 22.x
         uses: actions/setup-node@v4
         with:
-          node-version: '16'
+          node-version: '22'
       - name: apt-get install
         run: sudo apt-get install build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev
+      - run: corepack enable
       - run: yarn
       - run: yarn test
       - name: yarn semantic-release

.gitignore

@@ -9,3 +9,6 @@ gcp-credentials.json
 *-subset.js
 *-subset.d.ts
 *.generated.sql
+
+
+.yarn

.nvmrc

@@ -0,0 +1 @@
+v22

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

bin/automated-update.js

@@ -7,6 +7,8 @@ const {BigQuery} = require('@google-cloud/bigquery')
 
 const {getEntity} = require('../lib/')
 
+const bigQuery = new BigQuery()
+
 const HA_REQUESTS_TABLE_REGEX = /`httparchive\.requests\.\w+`/g
 const HA_LH_TABLE_REGEX = /`httparchive\.lighthouse\.\w+`/g
 const LH_3P_TABLE_REGEX = /`lighthouse-infrastructure\.third_party_web\.\w+`/g
@@ -81,7 +83,7 @@ async function getTargetDatasetDate() {
 }
 
 const getQueryResultStream = async query => {
-  const [job] = await new BigQuery().createQueryJob({
+  const [job] = await bigQuery.createQueryJob({
     query,
     location: 'US',
     useQueryCache: false,
@@ -96,8 +98,10 @@ const getJSONStringTransformer = rowCounter => {
   return new Transform({
     objectMode: true,
     transform(row, _, callback) {
-      const prefix = rowCounter === undefined ? '' : !rowCounter++ ? '[\n' : ',\n'
-      callback(null, prefix + JSON.stringify(row))
+      const toJSONArrayString = rowCounter !== undefined
+      const prefix = toJSONArrayString ? (!rowCounter++ ? '[\n' : ',\n') : ''
+      const suffix = toJSONArrayString ? '' : '\n'
+      callback(null, prefix + JSON.stringify(row) + suffix)
     },
   })
 }
@@ -114,6 +118,23 @@ const EntityCanonicalDomainTransformer = new Transform({
   },
 })
 
+const getThirdPartyWebTable = async tableName => {
+  const thirdPartyWebDataset = bigQuery.dataset('third_party_web', {
+    projectId: process.env.OVERRIDE_LH_PROJECT,
+  })
+  const thirdPartyWebTable = thirdPartyWebDataset.table(tableName)
+  const thirdPartyWebTableExits = await thirdPartyWebTable.exists()
+  if (thirdPartyWebTableExits) return thirdPartyWebTable
+  const [table] = await thirdPartyWebDataset.createTable(tableName, {
+    schema: [
+      {name: 'domain', type: 'STRING'},
+      {name: 'canonicalDomain', type: 'STRING'},
+      {name: 'category', type: 'STRING'},
+    ],
+  })
+  return table
+}
+
 async function main() {
   const {dateStringUnderscore, dateStringHypens} = await getTargetDatasetDate()
 
@@ -153,20 +174,17 @@ async function main() {
         .pipe(observedDomainsFileWriterStream)
 
       // Observed domain entity mapping table pipe
-      const thirdPartyWebTableWriterStream = new BigQuery()
-        .dataset('third_party_web')
-        .table(dateStringUnderscore)
-        .createWriteStream({
-          schema: [
-            {name: 'domain', type: 'STRING'},
-            {name: 'canonicalDomain', type: 'STRING'},
-            {name: 'category', type: 'STRING'},
-          ],
-        })
+      const thirdPartyWebTableWriterStream = await getThirdPartyWebTable(dateStringUnderscore).then(
+        table =>
+          table.createWriteStream({
+            sourceFormat: 'NEWLINE_DELIMITED_JSON',
+          })
+      )
+
       resultsStream
         // map observed domain to entity
         .pipe(EntityCanonicalDomainTransformer)
-        // stringify json
+        // stringify json with new line delimiter
         .pipe(getJSONStringTransformer())
         // write to thrid_party_web table
         .pipe(thirdPartyWebTableWriterStream)
@@ -182,8 +200,7 @@ async function main() {
       )
     },
     deleteFn: async () => {
-      const bqClient = new BigQuery()
-      await bqClient
+      await bigQuery
         .dataset('third_party_web')
         .table(dateStringUnderscore)
         .delete()

package.json

@@ -24,7 +24,7 @@
   },
   "license": "MIT",
   "devDependencies": {
-    "@google-cloud/bigquery": "^5.10.0",
+    "@google-cloud/bigquery": "^7.9.0",
     "chart.js": "^2.9.4",
     "chartjs-node-canvas": "^3.2.0",
     "colors": "^1.4.0",
@@ -41,5 +41,6 @@
       "httparchive-nostats-subset": "./lib/subsets/httparchive-nostats.js",
       "httparchive-subset": "./lib/subsets/httparchive.js"
     }
-  }
+  },
+  "packageManager": "[email protected]"
 }