refactor: include artifacts (#169)

* refactor: include artifacts
forensicanalysis · Oct 19, 2024 · cffa0e3 · cffa0e3
1 parent caade8d
commit cffa0e3
Show file tree

Hide file tree

Showing 21 changed files with 3,020 additions and 40 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,6 +16,15 @@ jobs:
       - run: make install generate-win fmt_linux
       - run: git diff --exit-code
 
+  validate:
+    name: validate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v5
+        with: { go-version: '1.23' }
+      - uses: actions/checkout@v4
+      - run: make validate
+
   lint:
     name: lint
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,3 @@
-# config
-config/artifacts/
-
 # go
 vendor
 

diff --git a/Makefile b/Makefile
@@ -49,13 +49,17 @@ test-coverage:
 	go tool cover -func=coverage.out
 	go tool cover -html=coverage.out
 
+.PHONY: validate
+validate:
+	@echo "Validating..."
+	cd tools/artifactvalidator && go build -o ../../build/bin/artifactvalidator .
+	./build/bin/artifactvalidator -entrypoints=DefaultCollection1 config/artifacts/*.yaml
+
 .PHONY: generate
 generate:
 	@echo "Generating..."
 	go install golang.org/x/tools/cmd/[email protected]
 	go install github.com/forensicanalysis/go-resources/cmd/[email protected]
-	rm -rf config/artifacts
-	git clone https://github.com/forensicanalysis/artifacts.git config/artifacts
 	go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
 	resources -package assets -output assets/bin.generated.go config/bin/*
 
@@ -68,21 +72,6 @@ generate-win: generate
 	rsrc -arch amd64 -manifest build/win/artifactcollector.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.user.syso
 	rsrc -arch 386 -manifest build/win/artifactcollector32.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.user.syso
 
-.PHONY: build
-build: generate
-	@echo "Building..."
-	go build -o build/bin/artifactcollector .
-
-.PHONY: build-linux
-build-linux: generate
-	@echo "Building for Linux..."
-	GOOS=linux GOARCH=amd64 go build -o build/bin/artifactcollector-linux .
-
-.PHONY: build-darwin
-build-darwin: generate
-	@echo "Building for macOS..."
-	GOOS=darwin GOARCH=amd64 go build -o build/bin/artifactcollector-darwin .
-
 .PHONY: build-win
 build-win: generate-win
 	@echo "Building for Windows..."

diff --git a/README.md b/README.md
@@ -61,18 +61,18 @@ The zip file contains the results of the extraction and needs to be transferred
 ## Build your own artifactcollector
 
 1. Clone the repository: `git clone https://github.com/forensicanalysis/artifactcollector`.
-2. Add artifact definition yaml files as needed in `config/artifacts`. Do not edit the
-   artifact definitions, as they will be overwritten.
+2. Add and edit artifact definition yaml files as needed in `config/artifacts`.
 3. Edit `config/ac.yaml` and add the artifacts you want to collect.
 4. On windows, you can move the syso into the root folder (e.g. `cp resources\artifactcollector.syso .`)
    to enable the icon for the executable and the UAC popup.
-5. Run `make build` to generate the artifactcollector binary.
+5. Run `go build .` to generate the artifactcollector binary.
+   1. You can also use `GOOS=windows GOARCH=amd64 go build -o artifactcollector.exe .` to cross-compile for Windows.
 
 ## Embed binaries
 
 Binaries can be added to `config/bin` and then included into the artifactcollector
-in the `make build` step. Additionally, a corresponding COMMAND artifact like
-the following is required.
+in the `go build` step. Additionally, a corresponding COMMAND artifact like
+the following is required:
 
 ```yaml
 name: Autoruns
@@ -84,12 +84,19 @@ sources:
 supported_os: [ Windows ]
 ```
 
-The command output to stdout and stderr is saved, but generated
-files are not collected.
+The command output to stdout and stderr is saved, but generated files are not collected.
+
+## Acknowledgement
+
+The artifactcollector uses on the following great projects:
+
+- [config/artifacts](config/artifacts) is based on the awesome [Forensic Artifacts](https://github.com/ForensicArtifacts/artifacts) project.
+- [doublestar](doublestar) is based on [Bob Matcuk's](https://github.com/bmatcuk) great [doublestar](https://github.com/bmatcuk/doublestar) package.
+- [store/aczip](store/aczip) and [build/go](build/go) contain code from the Go standard library.
 
 ## License
 
 Most of the artifactcollector is licensed under the MIT License. See [MIT license](LICENSE) for the full license text.
 
 The directories [store/aczip](store/aczip) and [build/go](build/go) contain code from the Go standard library
-which is licensed under the [BSD-3-Clause license](LICENSE-BSD).
+which is licensed under the [BSD-3-Clause license](LICENSE-BSD).
diff --git a/assets/artifacts.generated.go b/assets/artifacts.generated.go
diff --git a/build/win2k/Dockerfile b/build/win2k/Dockerfile
@@ -7,8 +7,6 @@ WORKDIR /repo
 RUN go install golang.org/x/tools/cmd/[email protected]
 RUN go install github.com/forensicanalysis/go-resources/cmd/[email protected]
 RUN go install github.com/akavel/[email protected]
-RUN rm -rf config/artifacts
-RUN git clone https://github.com/forensicanalysis/artifacts.git config/artifacts
 RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
 RUN resources -package assets -output assets/bin.generated.go config/bin/*
 RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso

diff --git a/build/winxp/Dockerfile b/build/winxp/Dockerfile
@@ -7,8 +7,6 @@ WORKDIR /repo
 RUN go install golang.org/x/tools/cmd/[email protected]
 RUN go install github.com/forensicanalysis/go-resources/cmd/[email protected]
 RUN go install github.com/akavel/[email protected]
-RUN rm -rf config/artifacts
-RUN git clone https://github.com/forensicanalysis/artifacts.git config/artifacts
 RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
 RUN resources -package assets -output assets/bin.generated.go config/bin/*
 RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso

diff --git a/config/ac.yaml b/config/ac.yaml
@@ -1 +1,4 @@
 artifacts: ["DefaultCollection1"]  # artifact definitions to collect
+user: false # optional, if true, do not request admin permissions
+case: "" # optional case name
+output_dir: "" # optional output directory
diff --git a/config/artifacts/README.md b/config/artifacts/README.md
@@ -0,0 +1,14 @@
+## Artifact Definitions
+
+The artifactcollector uses yaml files to define forensic artifacts it can collect.
+
+The yaml files are based on the [ForensicArtifacts/artifacts](https://github.com/ForensicArtifacts/artifacts)
+repository, but with the following major changes:
+
+- `provides` on source level are added to enable extraction of parameters
+- All source types are distinctly defined, including the `DIRECTORY` type.
+- Parameter expansion and globing is defined, including `**`.
+- Inconsistent trailing `\*` in REGISTRY_KEYs are removed.
+
+The [Style Guide](style_guide.md) describes the full specification of the artifact definitions 
+how they are used in the artifactcollector.
diff --git a/config/artifacts/collections.yaml b/config/artifacts/collections.yaml
@@ -0,0 +1,168 @@
+# Predefined opinionated collections
+
+name: DefaultCollection1
+doc: Predefined opinionated collections
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - 'FOR500'
+        - 'WindowsComputerName'
+        - 'WindowsEventLogs'
+        - 'WindowsHotfixes'
+        - 'WindowsNetworkInterfaceInformation'
+        - 'WindowsPersistence'
+        - 'WindowsRunKeys'
+        - 'WindowsServices'
+        - 'WindowsUninstallKeys'
+        - 'WindowsUSBInformation'
+    supported_os: [ Windows ]
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # - 'BrowserCache'
+        - 'BrowserHistory'
+        - 'LinuxIPTablesRulesCommand'
+        - 'LinuxAtJobsFiles'
+        - 'LinuxAuditLogFiles'
+        - 'LinuxCronTabFiles'
+        - 'LinuxHostnameFile'
+    supported_os: [ Linux ]
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # - 'BrowserCache'
+        - 'BrowserHistory'
+        - 'MacOSAtJobsFile'
+        - 'MacOSAuditLogFiles'
+        - 'MacOSBashHistoryFile'
+        - 'MacOSCronTabFile'
+        - 'MacOSHostsFile'
+        - 'MacOSLastlogFile'
+        - 'MacOSMiscLogFiles'
+        - 'MacOSRecentItemsFiles'
+        - 'MacOSSystemLogFiles'
+        - 'MacOSUserTrashFiles'
+    supported_os: [ Darwin ]
+supported_os: [ Darwin,Linux,Windows ]
+---
+# Artifacts from the SANS FOR500 course
+
+name: FOR500
+doc: Windows Forensic Analysis
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsBrowserArtifacts
+        - WindowsProgramExecution
+        - WindowsDeletedFiles
+        - WindowsNetworkActivity
+        # - WindowsFileOpening
+        - AccountUsage
+        - ExternalDevice
+supported_os: [ Windows ]
+---
+name: WindowsBrowserArtifacts
+doc: WindowsBrowserArtifacts
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsOpenSaveMRU
+        - WindowsOpenSavePidlMRU
+        # EmailAttachments
+        # SkypeMainDirectory is only for macos
+        # BrowserCache not collected by default
+        - BrowserHistory
+        # AdsZoneIdentifier
+supported_os: [ Windows ]
+---
+name: WindowsProgramExecution
+doc: Program Execution
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # UserAssist
+        - WindowsActivitiesCacheDatabase
+        - WindowsMostRecentApplication
+        - WindowsAppCompatCache # Shimcache
+        # JumpLists
+        - WindowsAMCacheHveFile
+        - WindowsSystemResourceUsageMonitorDatabaseFile
+        # BAM/DAM
+        # LastVisitedMRU
+        - WindowsPrefetchFiles
+supported_os: [ Windows ]
+---
+name: WindowsDeletedFiles
+doc: Deleted Files
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # ACMRU
+        # Thumbcache
+        # Thumbs.db
+        # IEEdgeFile -> WindowsBrowserArtifacts
+        # WordWheelQuery
+        - WindowsRecycleBin
+        # LastVisitedMRU -> WindowsProgramExecution
+supported_os: [ Windows ]
+---
+name: WindowsNetworkActivity
+doc: Network Activity
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsTimezone
+        - InternetExplorerCookiesFile
+        # NetworkHistory
+        # WLANEventLog
+        # BrowserSearchTerms -> WindowsBrowserArtifacts
+        # WindowsSystemResourceUsageMonitorDatabaseFile -> WindowsProgramExecution
+supported_os: [ Windows ]
+# ---
+# name: WindowsFileOpening
+# doc: File Opening
+# sources:
+# - type: ARTIFACT_GROUP
+#   attributes:
+#     names:
+#       # WindowsOpenSaveMRU -> WindowsBrowserArtifacts
+#       # RecentFiles
+#       # JumpLists -> WindowsProgramExecution
+#       # ShellBags
+#       # LNKFiles
+#       # WindowsPrefetchFiles -> WindowsProgramExecution
+#       # LastVisitedMRU -> WindowsProgramExecution
+#       # IEEdgeFile -> WindowsBrowserArtifacts
+#       # OfficeRecentFiles
+# supported_os: [Windows]
+---
+name: AccountUsage
+doc: Account Usage
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        - WindowsSystemRegistryFiles
+        - WindowsXMLEventLogSecurityFile
+supported_os: [ Windows ]
+---
+name: ExternalDevice
+doc: External Device
+sources:
+  - type: ARTIFACT_GROUP
+    attributes:
+      names:
+        # KeyIdentification
+        - WindowsSetupApiLogs
+        # User
+        # PnPEvents
+        # VolumeSerialNumber
+        # DriverLetter
+        # LNKFiles -> WindowsFileOpening
+supported_os: [ Windows ]
diff --git a/config/artifacts/linux.yaml b/config/artifacts/linux.yaml
@@ -0,0 +1,65 @@
+# Linux specific artifacts.
+
+name: LinuxIPTablesRulesCommand
+doc: List IPTables rules.
+sources:
+  - type: COMMAND
+    attributes:
+      args: [ "-L", "-n", "-v" ]
+      cmd: /sbin/iptables
+supported_os: [ Linux ]
+---
+name: LinuxAtJobsFiles
+doc: Linux at jobs.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/var/spool/at/*' ] }
+supported_os: [ Linux ]
+---
+name: LinuxAuditLogFiles
+doc: Linux audit log files.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/var/log/audit/*' ] }
+supported_os: [ Linux ]
+---
+name: LinuxCronTabFiles
+doc: Crontab files.
+sources:
+  - type: FILE
+    attributes:
+      paths:
+        - '/etc/crontab'
+        - '/etc/cron.d/*'
+        - '/var/spool/cron/**'
+supported_os: [ Linux ]
+---
+name: LinuxHostnameFile
+doc: Linux hostname file.
+sources:
+  - type: FILE
+    attributes: { paths: [ '/etc/hostname' ] }
+supported_os: [ Linux ]
+---
+name: LinuxPasswdFile
+doc: |
+  Linux passwd file.
+
+  A passwd file consist of colon separated values in the format:
+  username:password:uid:gid:full name:home directory:shell
+sources:
+  - type: FILE
+    attributes: { paths: [ '/etc/passwd' ] }
+    provides:
+      - key: users.homedir
+        regex: '.*:(.*?):.*'
+supported_os: [ Linux ]
+---
+name: LinuxHomePath
+doc: Users directories in /home
+sources:
+  - type: PATH
+    attributes: { paths: [ '/home/*' ] }
+    provides:
+      - key: users.homedir
+supported_os: [ Linux ]