refactor: include artifacts

.github/workflows/ci.yml

@@ -16,6 +16,15 @@ jobs:
       - run: make install generate-win fmt
       - run: git diff --exit-code
 
+  validate:
+    name: validate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v5
+        with: { go-version: '1.23' }
+      - uses: actions/checkout@v4
+      - run: make validate
+
   lint:
     name: lint
     runs-on: ubuntu-latest

.gitignore

@@ -1,6 +1,3 @@
-# config
-config/artifacts/
-
 # go
 vendor
 

Makefile

@@ -39,13 +39,17 @@ test-coverage:
 	go tool cover -func=coverage.out
 	go tool cover -html=coverage.out
 
+.PHONY: validate
+validate:
+	@echo "Validating..."
+	cd tools/artifactvalidator && go build -o ../../build/bin/artifactvalidator .
+	./build/bin/artifactvalidator -entrypoints=DefaultCollection1 config/artifacts/*.yaml
+
 .PHONY: generate
 generate:
 	@echo "Generating..."
 	go install golang.org/x/tools/cmd/[email protected]
 	go install github.com/forensicanalysis/go-resources/cmd/[email protected]
-	rm -rf config/artifacts
-	git clone https://github.com/forensicanalysis/artifacts.git config/artifacts
 	go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
 	resources -package assets -output assets/bin.generated.go config/bin/*
 

config/artifacts/README.md

@@ -0,0 +1,14 @@
+## Artifact Definitions
+
+The artifactcollector uses yaml files to define forensic artifacts it can collect.
+
+The yaml files are based on the [ForensicArtifacts/artifacts](https://github.com/ForensicArtifacts/artifacts)
+repository, but with the following major changes:
+
+- `provides` on source level are added to enable extraction of parameters
+- All source types are distinctly defined, including the `DIRECTORY` type.
+- Parameter expansion and globing is defined, including `**`.
+- Inconsistent trailing `\*` in REGISTRY_KEYs are removed.
+
+The [Style Guide](style_guide.md) describes the full specification of the artifact definitions 
+how they are used in the artifactcollector.

config/artifacts/collections.yaml

@@ -0,0 +1,168 @@
+# Predefined opinionated collections
+
+name: DefaultCollection1
+doc: Predefined opinionated collections
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - 'FOR500'
+        - 'WindowsComputerName'
+        - 'WindowsEventLogs'
+        - 'WindowsHotfixes'
+        - 'WindowsNetworkInterfaceInformation'
+        - 'WindowsPersistence'
+        - 'WindowsRunKeys'
+        - 'WindowsServices'
+        - 'WindowsUninstallKeys'
+        - 'WindowsUSBInformation'
+    supported_os: [ Windows ]
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # - 'BrowserCache'
+        - 'BrowserHistory'
+        - 'LinuxIPTablesRulesCommand'
+        - 'LinuxAtJobsFiles'
+        - 'LinuxAuditLogFiles'
+        - 'LinuxCronTabFiles'
+        - 'LinuxHostnameFile'
+    supported_os: [ Linux ]
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # - 'BrowserCache'
+        - 'BrowserHistory'
+        - 'MacOSAtJobsFile'
+        - 'MacOSAuditLogFiles'
+        - 'MacOSBashHistoryFile'
+        - 'MacOSCronTabFile'
+        - 'MacOSHostsFile'
+        - 'MacOSLastlogFile'
+        - 'MacOSMiscLogFiles'
+        - 'MacOSRecentItemsFiles'
+        - 'MacOSSystemLogFiles'
+        - 'MacOSUserTrashFiles'
+    supported_os: [ Darwin ]
+supported_os: [ Darwin,Linux,Windows ]
+---
+# Artifacts from the SANS FOR500 course
+
+name: FOR500
+doc: Windows Forensic Analysis
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsBrowserArtifacts
+        - WindowsProgramExecution
+        - WindowsDeletedFiles
+        - WindowsNetworkActivity
+        # - WindowsFileOpening
+        - AccountUsage
+        - ExternalDevice
+supported_os: [ Windows ]
+---
+name: WindowsBrowserArtifacts
+doc: WindowsBrowserArtifacts
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsOpenSaveMRU
+        - WindowsOpenSavePidlMRU
+        # EmailAttachments
+        # SkypeMainDirectory is only for macos
+        # BrowserCache not collected by default
+        - BrowserHistory
+        # AdsZoneIdentifier
+supported_os: [ Windows ]
+---
+name: WindowsProgramExecution
+doc: Program Execution
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # UserAssist
+        - WindowsActivitiesCacheDatabase
+        - WindowsMostRecentApplication
+        - WindowsAppCompatCache # Shimcache
+        # JumpLists
+        - WindowsAMCacheHveFile
+        - WindowsSystemResourceUsageMonitorDatabaseFile
+        # BAM/DAM
+        # LastVisitedMRU
+        - WindowsPrefetchFiles
+supported_os: [ Windows ]
+---
+name: WindowsDeletedFiles
+doc: Deleted Files
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # ACMRU
+        # Thumbcache
+        # Thumbs.db
+        # IEEdgeFile -> WindowsBrowserArtifacts
+        # WordWheelQuery
+        - WindowsRecycleBin
+        # LastVisitedMRU -> WindowsProgramExecution
+supported_os: [ Windows ]
+---
+name: WindowsNetworkActivity
+doc: Network Activity
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsTimezone
+        - InternetExplorerCookiesFile
+        # NetworkHistory
+        # WLANEventLog
+        # BrowserSearchTerms -> WindowsBrowserArtifacts
+        # WindowsSystemResourceUsageMonitorDatabaseFile -> WindowsProgramExecution
+supported_os: [ Windows ]
+# ---
+# name: WindowsFileOpening
+# doc: File Opening
+# sources:
+# - type: ARTIFACT_GROUP
+#   attributes:
+#     names:
+#       # WindowsOpenSaveMRU -> WindowsBrowserArtifacts
+#       # RecentFiles
+#       # JumpLists -> WindowsProgramExecution
+#       # ShellBags
+#       # LNKFiles
+#       # WindowsPrefetchFiles -> WindowsProgramExecution
+#       # LastVisitedMRU -> WindowsProgramExecution
+#       # IEEdgeFile -> WindowsBrowserArtifacts
+#       # OfficeRecentFiles
+# supported_os: [Windows]
+---
+name: AccountUsage
+doc: Account Usage
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsSystemRegistryFiles
+        - WindowsXMLEventLogSecurityFile
+supported_os: [ Windows ]
+---
+name: ExternalDevice
+doc: External Device
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # KeyIdentification
+        - WindowsSetupApiLogs
+        # User
+        # PnPEvents
+        # VolumeSerialNumber
+        # DriverLetter
+        # LNKFiles -> WindowsFileOpening
+supported_os: [ Windows ]

config/artifacts/linux.yaml

@@ -0,0 +1,65 @@
+# Linux specific artifacts.
+
+name: LinuxIPTablesRulesCommand
+doc: List IPTables rules.
+sources:
+  - type: COMMAND
+    attributes:
+      args: [ "-L", "-n", "-v" ]
+      cmd: /sbin/iptables
+supported_os: [ Linux ]
+---
+name: LinuxAtJobsFiles
+doc: Linux at jobs.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/var/spool/at/*' ] }
+supported_os: [ Linux ]
+---
+name: LinuxAuditLogFiles
+doc: Linux audit log files.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/var/log/audit/*' ] }
+supported_os: [ Linux ]
+---
+name: LinuxCronTabFiles
+doc: Crontab files.
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/etc/crontab'
+        - '/etc/cron.d/*'
+        - '/var/spool/cron/**'
+supported_os: [ Linux ]
+---
+name: LinuxHostnameFile
+doc: Linux hostname file.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/etc/hostname' ] }
+supported_os: [ Linux ]
+---
+name: LinuxPasswdFile
+doc: |
+  Linux passwd file.
+
+  A passwd file consist of colon separated values in the format:
+  username:password:uid:gid:full name:home directory:shell
+sources:
+  - type: FILE
+    attributes: { paths: [ '/etc/passwd' ] }
+    provides:
+      - key: users.homedir
+        regex: '.*:(.*?):.*'
+supported_os: [ Linux ]
+---
+name: LinuxHomePath
+doc: Users directories in /home
+sources:
+  - type: PATH
+    attributes: { paths: [ '/home/*' ] }
+    provides:
+      - key: users.homedir
+supported_os: [ Linux ]

config/artifacts/macos.yaml

@@ -0,0 +1,96 @@
+# MacOS (Darwin) specific artifacts.
+
+name: MacOSAtJobsFile
+doc: MacOS at jobs
+sources:
+  - type: FILE
+    attributes: { paths: [ '/usr/lib/cron/jobs/*' ] }
+supported_os: [ Darwin ]
+---
+name: MacOSAuditLogFiles
+doc: Audit log files
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/private/var/audit/*'
+        - '/var/audit/*'
+supported_os: [ Darwin ]
+---
+name: MacOSBashHistoryFile
+doc: Terminal Commands History
+sources:
+  - type: FILE
+    attributes: { paths: [ '%%users.homedir%%/.bash_history' ] }
+supported_os: [ Darwin ]
+---
+name: MacOSCronTabFile
+doc: Cron tabs
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/etc/crontab'
+        - '/private/etc/crontab'
+        - '/usr/lib/cron/tabs/*'
+supported_os: [ Darwin ]
+---
+name: MacOSHostsFile
+doc: Hosts file
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/etc/hosts'
+        - '/private/etc/hosts'
+supported_os: [ Darwin ]
+---
+name: MacOSLastlogFile
+doc: Mac OS X lastlog file.
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/private/var/log/lastlog'
+        - '/var/log/lastlog'
+supported_os: [ Darwin ]
+---
+name: MacOSMiscLogFiles
+doc: Misc. Logs
+sources:
+  - type: FILE
+    attributes: { paths: [ '/Library/Logs/*' ] }
+supported_os: [ Darwin ]
+---
+name: MacOSRecentItemsFiles
+doc: Recent Items
+sources:
+  - type: FILE
+    attributes: { paths: [ '%%users.homedir%%/Library/Preferences/com.apple.recentitems.plist' ] }
+supported_os: [ Darwin ]
+---
+name: MacOSSystemLogFiles
+doc: System log files
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/private/var/log/*'
+        - '/var/log/*'
+supported_os: [ Darwin ]
+---
+name: MacOSUsersPath
+doc: Users directories in /Users
+sources:
+  - type: PATH
+    attributes: { paths: [ '/Users/*' ] }
+    provides:
+      - key: users.homedir
+supported_os: [ Darwin ]
+---
+name: MacOSUserTrashFiles
+doc: User Trash Folder
+sources:
+  - type: FILE
+    attributes: { paths: [ '%%users.homedir%%/.Trash/*' ] }
+supported_os: [ Darwin ]