feat: Track files referenced by other files to identify orphans

bids-validator/src/files/browser.ts

@@ -11,6 +11,7 @@ export class BIDSFileBrowser implements BIDSFile {
   name: string
   path: string
   parent: FileTree
+  viewed: boolean = false
 
   constructor(file: File, ignore: FileIgnoreRules, parent?: FileTree) {
     this.#file = file

bids-validator/src/files/deno.ts

@@ -26,6 +26,7 @@ export class BIDSFileDeno implements BIDSFile {
   parent: FileTree
   #fileInfo?: Deno.FileInfo
   #datasetAbsPath: string
+  viewed: boolean = false
 
   constructor(datasetPath: string, path: string, ignore: FileIgnoreRules, parent?: FileTree) {
     this.#datasetAbsPath = datasetPath

bids-validator/src/files/inheritance.ts

@@ -30,6 +30,7 @@ export function* walkBack(
         )
       })
       if (exactMatch) {
+        exactMatch.viewed = true
         yield exactMatch
       } else {
         console.warn(`
@@ -39,6 +40,7 @@ ${candidates.map((file) => `* ${file.path}`).join('\n')}
 `)
       }
     } else if (candidates.length === 1) {
+      candidates[0].viewed = true
       yield candidates[0]
     }
     if (!inherit) break

bids-validator/src/issues/datasetIssues.test.ts

@@ -27,6 +27,7 @@ Deno.test('DatasetIssues management class', async (t) => {
         ignored: false,
         stream: testStream,
         parent: root,
+        viewed: false,
       } as BIDSFile,
       {
         text,
@@ -40,6 +41,7 @@ Deno.test('DatasetIssues management class', async (t) => {
         severity: 'warning',
         reason: 'Readme borked',
         parent: root,
+        viewed: false,
       } as IssueFile,
     ]
     issues.add({ key: 'TEST_FILES_ERROR', reason: 'Test issue', files })

bids-validator/src/issues/list.ts

@@ -131,6 +131,15 @@ export const bidsIssues: IssueDefinitionRecord = {
     severity: 'error',
     reason: 'Empty files not allowed.',
   },
+  UNUSED_STIMULUS: {
+    severity: 'warning',
+    reason:
+      'There are files in the /stimuli directory that are not utilized in any _events.tsv file.',
+  },
+  SIDECAR_WITHOUT_DATAFILE: {
+    severity: 'error',
+    reason: 'A json sidecar file was found without a corresponding data file',
+  },
 }
 
 const hedIssues: IssueDefinitionRecord = {

bids-validator/src/schema/fixtures.test.ts

@@ -41,6 +41,7 @@ export const dataFile = {
   stream: new ReadableStream<Uint8Array>(),
   readBytes: nullReadBytes,
   parent: anatFileTree,
+  viewed: false,
 }
 
 anatFileTree.files = [
@@ -54,6 +55,7 @@ anatFileTree.files = [
     stream: new ReadableStream<Uint8Array>(),
     readBytes: async (size: number) => new TextEncoder().encode(await anatJson()),
     parent: anatFileTree,
+    viewed: false,
   },
 ]
 
@@ -70,6 +72,7 @@ subjectFileTree.files = [
     stream: new ReadableStream<Uint8Array>(),
     readBytes: async (size: number) => new TextEncoder().encode(await subjectJson()),
     parent: subjectFileTree,
+    viewed: false,
   },
 ]
 subjectFileTree.directories = [sessionFileTree]
@@ -84,6 +87,7 @@ stimuliFileTree.files = [...Array(10).keys()].map((i) => (
     stream: new ReadableStream<Uint8Array>(),
     readBytes: nullReadBytes,
     parent: stimuliFileTree,
+    viewed: false,
   }
 ))
 
@@ -97,6 +101,7 @@ rootFileTree.files = [
     stream: new ReadableStream<Uint8Array>(),
     readBytes: async (size: number) => new TextEncoder().encode(await rootJson()),
     parent: rootFileTree,
+    viewed: false,
   },
 ]
 rootFileTree.directories = [stimuliFileTree, subjectFileTree]

bids-validator/src/schema/walk.ts

@@ -25,6 +25,7 @@ function pseudoFile(dir: FileTree): BIDSFile {
     size: [...quickWalk(dir)].reduce((acc, file) => acc + file.size, 0),
     ignored: dir.ignored,
     parent: dir.parent as FileTree,
+    viewed: false,
     ...nullFile,
   }
 }

bids-validator/src/tests/simple-dataset.ts

@@ -17,6 +17,7 @@ anatFileTree.files = [
     stream: new ReadableStream<Uint8Array>(),
     readBytes: nullReadBytes,
     parent: anatFileTree,
+    viewed: false,
   },
 ]
 subjectFileTree.files = []
@@ -31,6 +32,7 @@ rootFileTree.files = [
     stream: new ReadableStream(),
     readBytes: nullReadBytes,
     parent: rootFileTree,
+    viewed: false,
   },
   {
     text,
@@ -41,6 +43,7 @@ rootFileTree.files = [
     stream: new ReadableStream(),
     readBytes: nullReadBytes,
     parent: rootFileTree,
+    viewed: false,
   },
   {
     text,
@@ -51,6 +54,7 @@ rootFileTree.files = [
     stream: new ReadableStream(),
     readBytes: nullReadBytes,
     parent: rootFileTree,
+    viewed: false,
   },
   {
     text,
@@ -61,6 +65,7 @@ rootFileTree.files = [
     stream: new ReadableStream(),
     readBytes: nullReadBytes,
     parent: rootFileTree,
+    viewed: false,
   },
 ]
 rootFileTree.directories = [subjectFileTree]

bids-validator/src/types/filetree.ts

@@ -16,7 +16,10 @@ export interface BIDSFile {
   text: () => Promise<string>
   // Read a range of bytes
   readBytes: (size: number, offset?: number) => Promise<Uint8Array>
+  // Access the parent directory
   parent: FileTree
+  // File has been viewed
+  viewed: boolean
 }
 
 export class FileTree {
@@ -43,7 +46,7 @@ export class FileTree {
       return false
     } else if (parts.length === 1) {
       return (
-        this.files.some((x) => x.name === parts[0]) ||
+        this.files.some((x) => (x.name === parts[0] && (x.viewed = true))) ||
         this.directories.some((x) => x.name === parts[0])
       )
     } else if (parts.length > 1) {

bids-validator/src/validators/bids.ts

@@ -13,6 +13,7 @@ import { filenameIdentify } from './filenameIdentify.ts'
 import { filenameValidate } from './filenameValidate.ts'
 import { DatasetIssues } from '../issues/datasetIssues.ts'
 import { emptyFile } from './internal/emptyFile.ts'
+import { sidecarWithoutDatafile, unusedStimulus } from './internal/unusedFile.ts'
 import { BIDSContext, BIDSContextDataset } from '../schema/context.ts'
 import { parseOptions } from '../setup/options.ts'
 import { hedValidate } from './hed.ts'
@@ -28,7 +29,10 @@ const perContextChecks: ContextCheckFunction[] = [
   hedValidate,
 ]
 
-const perDSChecks: DSCheckFunction[] = []
+const perDSChecks: DSCheckFunction[] = [
+  unusedStimulus,
+  sidecarWithoutDatafile,
+]
 
 /**
  * Full BIDS schema validation entrypoint

bids-validator/src/validators/internal/unusedFile.ts

@@ -0,0 +1,45 @@
+import { GenericSchema } from '../../types/schema.ts'
+import { BIDSFile, FileTree } from '../../types/filetree.ts'
+import { BIDSContextDataset } from '../../schema/context.ts'
+
+function* walkFileTree(fileTree?: FileTree): Generator<BIDSFile> {
+  if (!fileTree) {
+    return
+  }
+  for (const file of fileTree.files) {
+    if (!file.ignored) {
+      yield file
+    }
+  }
+  for (const dir of fileTree.directories) {
+    if (!dir.ignored) {
+      yield* walkFileTree(dir)
+    }
+  }
+}
+
+export async function unusedStimulus(
+  schema: GenericSchema,
+  dsContext: BIDSContextDataset,
+) {
+  const stimDir = dsContext.tree.directories.find((dir) => dir.name === 'stimuli')
+  const unusedStimuli = [...walkFileTree(stimDir)].filter((stimulus) => !stimulus.viewed)
+  if (unusedStimuli.length) {
+    dsContext.issues.addNonSchemaIssue('UNUSED_STIMULUS', unusedStimuli)
+  }
+}
+
+const standalone_json = ['dataset_description.json', 'genetic_info.json']
+
+export async function sidecarWithoutDatafile(
+  schema: GenericSchema,
+  dsContext: BIDSContextDataset,
+) {
+  const unusedSidecars = [...walkFileTree(dsContext.tree)].filter(
+    (file) => (!file.viewed && file.name.endsWith('.json') &&
+      !standalone_json.includes(file.name)),
+  )
+  if (unusedSidecars.length) {
+    dsContext.issues.addNonSchemaIssue('SIDECAR_WITHOUT_DATAFILE', unusedSidecars)
+  }
+}