From 6ee8310a86390af03047ddb87f6e33e39a46676b Mon Sep 17 00:00:00 2001 From: Jonas Plum Date: Sat, 19 Oct 2024 14:46:54 +0200 Subject: [PATCH] refactor: include resources (#171) * refactor: include resources --- Makefile | 7 +- README.md | 1 + build/win2k/Dockerfile | 10 +- build/winxp/Dockerfile | 10 +- tools/resources/README.md | 81 ++++++++++++ tools/resources/cmd.go | 80 ++++++++++++ tools/resources/resources.go | 176 +++++++++++++++++++++++++++ tools/resources/resources_test.go | 46 +++++++ tools/resources/testdata/12.bin | Bin 0 -> 4 bytes tools/resources/testdata/123.bin | Bin 0 -> 256 bytes tools/resources/testdata/patrick.txt | 2 + tools/resources/testdata/query.sql | 2 + tools/resources/testdata/test.txt | 1 + 13 files changed, 395 insertions(+), 21 deletions(-) create mode 100644 tools/resources/README.md create mode 100644 tools/resources/cmd.go create mode 100644 tools/resources/resources.go create mode 100644 tools/resources/resources_test.go create mode 100644 tools/resources/testdata/12.bin create mode 100644 tools/resources/testdata/123.bin create mode 100644 tools/resources/testdata/patrick.txt create mode 100644 tools/resources/testdata/query.sql create mode 100644 tools/resources/testdata/test.txt diff --git a/Makefile b/Makefile index 5004810..271ca72 100644 --- a/Makefile +++ b/Makefile @@ -59,9 +59,10 @@ validate: generate: @echo "Generating..." go install golang.org/x/tools/cmd/goimports@v0.1.7 - go install github.com/forensicanalysis/go-resources/cmd/resources@v0.4.0 - go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml - resources -package assets -output assets/bin.generated.go config/bin/* + cd tools/yaml2go && go build -o ../../build/bin/yaml2go . + ./build/bin/yaml2go config/ac.yaml config/artifacts/*.yaml + cd tools/resources && go build -o ../../build/bin/resources . + ./build/bin/resources -package assets -output assets/bin.generated.go config/bin/* .PHONY: generate-win generate-win: generate diff --git a/README.md b/README.md index 71075f4..e108e8f 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ The artifactcollector uses on the following great projects: - [config/artifacts](config/artifacts) is based on the awesome [Forensic Artifacts](https://github.com/ForensicArtifacts/artifacts) project. - [doublestar](doublestar) is based on [Bob Matcuk's](https://github.com/bmatcuk) great [doublestar](https://github.com/bmatcuk/doublestar) package. - [store/aczip](store/aczip) and [build/go](build/go) contain code from the Go standard library. +- [tools/resources](tools/resources) is based on [go-resources](https://github.com/omeid/go-resources). ## License diff --git a/build/win2k/Dockerfile b/build/win2k/Dockerfile index 7a9d40d..081dd51 100644 --- a/build/win2k/Dockerfile +++ b/build/win2k/Dockerfile @@ -4,15 +4,7 @@ COPY . /repo WORKDIR /repo -RUN go install golang.org/x/tools/cmd/goimports@v0.1.7 -RUN go install github.com/forensicanalysis/go-resources/cmd/resources@v0.4.0 -RUN go install github.com/akavel/rsrc@v0.10.2 -RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml -RUN resources -package assets -output assets/bin.generated.go config/bin/* -RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso -RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.syso -RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.user.syso -RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.user.syso +RUN make generate-win RUN mv build/win/artifactcollector32.syso artifactcollector.syso FROM golang:1.2.2-cross diff --git a/build/winxp/Dockerfile b/build/winxp/Dockerfile index 2699e98..2f929f9 100644 --- a/build/winxp/Dockerfile +++ b/build/winxp/Dockerfile @@ -4,15 +4,7 @@ COPY . /repo WORKDIR /repo -RUN go install golang.org/x/tools/cmd/goimports@v0.1.7 -RUN go install github.com/forensicanalysis/go-resources/cmd/resources@v0.4.0 -RUN go install github.com/akavel/rsrc@v0.10.2 -RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml -RUN resources -package assets -output assets/bin.generated.go config/bin/* -RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso -RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.syso -RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.user.syso -RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.user.syso +RUN make generate-win RUN mv build/win/artifactcollector32.syso artifactcollector.syso FROM golang:1.9.7 diff --git a/tools/resources/README.md b/tools/resources/README.md new file mode 100644 index 0000000..09c8cb5 --- /dev/null +++ b/tools/resources/README.md @@ -0,0 +1,81 @@ +# go-resources + +> [!IMPORTANT] +> You should use the `embed` package instead of this tool. +> See https://pkg.go.dev/embed for more information. +> +> Only use this tool if you need to support older (< 1.16) versions of Go. + +go-resources is a tool to embed files into Go source code. + +## Installation + +```sh +go install github.com/forensicanalysis/go-resources@latest +``` + +## Usage + +```sh +resources -h +Usage resources: + -output filename + filename to write the output to + -package name + name of the package to generate (default "main") + -tag tag + tag to use for the generated package (default no tag) + -trim prefix + path prefix to remove from the resulting file path in the virtual filesystem + -var name + name of the variable to assign the virtual filesystem to (default "FS") +``` + +## Optimization + +Generating resources result in a very high number of lines of code, 1MB +of resources result about 5MB of code at over 87,000 lines of code. This +is caused by the chosen representation of the file contents within the +generated file. + +Instead of a (binary) string, `resources` transforms each file into an +actual byte slice. For example, a file with content `Hello, world!` will +be represented as follows: + +``` go +var FS = map[string][]byte{ + "/hello.txt": []byte{ + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, + 0x21, + }, +} +``` + +While this seems wasteful, the compiled binary is not really affected. +_If you add 1MB of resources, your binary will increase 1MB as well_. + +However, compiling this many lines of code takes time and slows down the +compiler. To avoid recompiling the resources every time and leverage the +compiler cache, generate your resources into a standalone package and +then import it, this will allow for faster iteration as you don't have +to wait for the resources to be compiled with every change. + +``` sh +mkdir -p assets +resources -var=FS -package=assets -output=assets/assets.go your/files/here +``` + +``` go +package main + +import "importpath/to/assets" + +func main() { + data, ok := assets.FS["your/files/here"] + // ... +} +``` + +## Credits + +This is a fork of https://github.com/omeid/go-resources diff --git a/tools/resources/cmd.go b/tools/resources/cmd.go new file mode 100644 index 0000000..eb41e52 --- /dev/null +++ b/tools/resources/cmd.go @@ -0,0 +1,80 @@ +// Unfancy resources embedding with Go. + +package main + +import ( + "flag" + "log" + "os" + "path/filepath" + "strings" + "time" +) + +var ( + pkgName = "main" + varName = "FS" + tag = "" + out = "" + trimPath = "" +) + +type nope struct{} + +func main() { + t0 := time.Now() + + flag.StringVar(&pkgName, "package", pkgName, "`name` of the package to generate") + flag.StringVar(&varName, "var", varName, "`name` of the variable to assign the virtual filesystem to") + flag.StringVar(&tag, "tag", tag, "`tag` to use for the generated package (default no tag)") + flag.StringVar(&out, "output", out, "`filename` to write the output to") + flag.StringVar(&trimPath, "trim", trimPath, "path `prefix` to remove from the resulting file path in the virtual filesystem") + flag.Parse() + + if out == "" { + flag.PrintDefaults() + log.Fatal("-output is required.") + } + + config := Config{ + Pkg: pkgName, + Var: varName, + Tag: tag, + } + + res := New() + res.Config = config + + files := make(map[string]nope) + + for _, g := range flag.Args() { + matches, err := filepath.Glob(g) + if err != nil { + log.Fatal(err) + } + + for _, m := range matches { + info, err := os.Stat(m) + + if !os.IsNotExist(err) && !info.IsDir() { + files[m] = nope{} + } + } + } + + for path := range files { + name := filepath.ToSlash(path) + name = strings.TrimPrefix(name, trimPath) + + err := res.AddFile(name, path) + if err != nil { + log.Fatal(err) + } + } + + if err := res.Write(out); err != nil { + log.Fatal(err) + } + + log.Printf("Finished in %v. Wrote %d resources to %s", time.Since(t0), len(files), out) +} diff --git a/tools/resources/resources.go b/tools/resources/resources.go new file mode 100644 index 0000000..77a8973 --- /dev/null +++ b/tools/resources/resources.go @@ -0,0 +1,176 @@ +// Package resources provides unfancy resources embedding with Go. +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "text/template" +) + +// File mimicks the os.File and http.File interface. +type File interface { + io.Reader + Stat() (os.FileInfo, error) +} + +// New creates a new Package. +func New() *Package { + return &Package{ + Config: Config{ + Pkg: "resources", + Var: "FS", + }, + Files: make(map[string]File), + } +} + +// Config defines some details about the output file. +type Config struct { + Pkg string // Pkg holds the package name + Var string // Var holds the variable name for the virtual filesystem + Tag string // Tag may hold an optional build tag, unless empty +} + +// A Package describes a collection of files and how they should be transformed +// to an output. +type Package struct { + Config + Files map[string]File +} + +// Add a file to the package at the give path. +func (p *Package) Add(name string, file File) error { + name = filepath.ToSlash(name) + p.Files[name] = file + + return nil +} + +// AddFile is a helper function that adds the files from the path into the +// package under the path file. +func (p *Package) AddFile(name, path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + return p.Add(name, f) +} + +// Build compiles the package and writes it into an io.Writer. +func (p *Package) Build(out io.Writer) error { + return pkg.Execute(out, p) +} + +// Write builds the package (via Build) and writes the output the file +// given by the path argument. +func (p *Package) Write(path string) error { + err := os.MkdirAll(filepath.Dir(path), 0700) + if err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + + defer func() { + err := f.Close() + if err != nil { + log.Panicf("Failed to close file: %s", err) + } + }() + + return p.Build(f) +} + +var ( + // Template. + pkg *template.Template + + // BlockWidth allows to adjust the number of bytes per line in the generated file. + BlockWidth = 12 +) + +func reader(input io.Reader, indent int) (string, error) { + var ( + buff bytes.Buffer + strbuf strings.Builder + isString bool + err error + curblock = 0 + linebreak = "\n" + strings.Repeat("\t", indent) + ) + + b := make([]byte, BlockWidth) + isString = true + + for n, e := input.Read(b); e == nil; n, e = input.Read(b) { + for i := range n { + if isString { + if isGoASCII(rune(b[i])) { + strbuf.WriteByte(b[i]) + } else { + isString = false + } + } + + _, e = fmt.Fprintf(&buff, "0x%02x,", b[i]) + if e != nil { + err = e + + break + } + + curblock++ + if curblock < BlockWidth { + buff.WriteRune(' ') + + continue + } + + buff.WriteString(linebreak) + + curblock = 0 + } + } + + if isString { + return "[]byte(`" + strbuf.String() + "`),", err + } + + return "{" + linebreak + buff.String() + "\n" + strings.Repeat("\t", indent-1) + "},", err +} + +func isGoASCII(b rune) bool { + if ((' ' <= b && b <= '~') || b == '\n' || b == '\t' || b == '\r') && b != '`' { + return true + } + + return false +} + +func init() { + pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(fileTemplate)) + pkg = template.Must(pkg.New("pkg").Parse(pkgTemplate)) +} + +const fileTemplate = `{{ reader . 4 }}` + +const pkgTemplate = `// Code generated by github.com/forensicanalysis/go-resources. DO NOT EDIT. + +{{ if .Tag }}// +build {{ .Tag }} + +{{ end }} +package {{ .Pkg }} + +var {{ .Var }} = map[string][]byte{ {{range $path, $file := .Files }} + "/{{ $path }}": {{ template "file" $file }}{{ end }} +} +` diff --git a/tools/resources/resources_test.go b/tools/resources/resources_test.go new file mode 100644 index 0000000..f44d5cc --- /dev/null +++ b/tools/resources/resources_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/forensicanalysis/go-resources/testdata/generated" +) + +//go:generate go build -o testdata/resources . +//go:generate testdata/resources -package generated -output testdata/generated/store_prod.go testdata/*.txt testdata/*.sql testdata/*.bin + +func TestGenerated(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + }{ + {name: "test.txt"}, + {name: "patrick.txt"}, + {name: "query.sql"}, + {name: "123.bin"}, + {name: "12.bin"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + content, ok := generated.FS["/testdata/"+tt.name] + + if !ok { + t.Fatalf("expected no error opening file") + } + + data, err := os.ReadFile(filepath.Join("testdata", tt.name)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(content, data) { + t.Errorf("expected to find snippet '%x', got: '%x'", data, content) + } + }) + } +} diff --git a/tools/resources/testdata/12.bin b/tools/resources/testdata/12.bin new file mode 100644 index 0000000000000000000000000000000000000000..eaf36c1daccfdf325514461cd1a2ffbc139b5464 GIT binary patch literal 4 LcmZQzWMT#Y01f~L literal 0 HcmV?d00001 diff --git a/tools/resources/testdata/123.bin b/tools/resources/testdata/123.bin new file mode 100644 index 0000000000000000000000000000000000000000..c86626638e0bc8cf47ca49bb1525b40e9737ee64 GIT binary patch literal 256 zcmV+b0ssC00RjUA1qKHQ2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0 zEiNxGF)}kWH8wXmIXXK$Jw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRG6Sz23MU0z>c zVPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^? zm6n&7nVOrNot~edp`xRtrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T z$;!*j&Cbuz(bCh@)z;V8+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0( G{r>;0_J4r@ literal 0 HcmV?d00001 diff --git a/tools/resources/testdata/patrick.txt b/tools/resources/testdata/patrick.txt new file mode 100644 index 0000000..dcfa344 --- /dev/null +++ b/tools/resources/testdata/patrick.txt @@ -0,0 +1,2 @@ +is this is test.txt? +no, this is patrick! diff --git a/tools/resources/testdata/query.sql b/tools/resources/testdata/query.sql new file mode 100644 index 0000000..c0307e9 --- /dev/null +++ b/tools/resources/testdata/query.sql @@ -0,0 +1,2 @@ +create table if not exist "files"; +drop table "files"; diff --git a/tools/resources/testdata/test.txt b/tools/resources/testdata/test.txt new file mode 100644 index 0000000..66e6110 --- /dev/null +++ b/tools/resources/testdata/test.txt @@ -0,0 +1 @@ +this is test.txt