refactor: include resources (#171)

* refactor: include resources

Makefile

@@ -59,9 +59,10 @@ validate:
 generate:
 	@echo "Generating..."
 	go install golang.org/x/tools/cmd/[email protected]
-	go install github.com/forensicanalysis/go-resources/cmd/[email protected]
-	go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
-	resources -package assets -output assets/bin.generated.go config/bin/*
+	cd tools/yaml2go && go build -o ../../build/bin/yaml2go .
+	./build/bin/yaml2go config/ac.yaml config/artifacts/*.yaml
+	cd tools/resources && go build -o ../../build/bin/resources .
+	./build/bin/resources -package assets -output assets/bin.generated.go config/bin/*
 
 .PHONY: generate-win
 generate-win: generate

README.md

@@ -93,6 +93,7 @@ The artifactcollector uses on the following great projects:
 - [config/artifacts](config/artifacts) is based on the awesome [Forensic Artifacts](https://github.com/ForensicArtifacts/artifacts) project.
 - [doublestar](doublestar) is based on [Bob Matcuk's](https://github.com/bmatcuk) great [doublestar](https://github.com/bmatcuk/doublestar) package.
 - [store/aczip](store/aczip) and [build/go](build/go) contain code from the Go standard library.
+- [tools/resources](tools/resources) is based on [go-resources](https://github.com/omeid/go-resources).
 
 ## License
 

build/win2k/Dockerfile

@@ -4,15 +4,7 @@ COPY . /repo
 
 WORKDIR /repo
 
-RUN go install golang.org/x/tools/cmd/[email protected]
-RUN go install github.com/forensicanalysis/go-resources/cmd/[email protected]
-RUN go install github.com/akavel/[email protected]
-RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
-RUN resources -package assets -output assets/bin.generated.go config/bin/*
-RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso
-RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.syso
-RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.user.syso
-RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.user.syso
+RUN make generate-win
 RUN mv build/win/artifactcollector32.syso artifactcollector.syso
 
 FROM golang:1.2.2-cross

build/winxp/Dockerfile

@@ -4,15 +4,7 @@ COPY . /repo
 
 WORKDIR /repo
 
-RUN go install golang.org/x/tools/cmd/[email protected]
-RUN go install github.com/forensicanalysis/go-resources/cmd/[email protected]
-RUN go install github.com/akavel/[email protected]
-RUN go run tools/yaml2go/main.go config/ac.yaml config/artifacts/*.yaml
-RUN resources -package assets -output assets/bin.generated.go config/bin/*
-RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.syso
-RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.syso
-RUN rsrc -arch amd64 -manifest build/win/artifactcollector.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector.user.syso
-RUN rsrc -arch 386 -manifest build/win/artifactcollector32.exe.user.manifest -ico build/win/artifactcollector.ico -o build/win/artifactcollector32.user.syso
+RUN make generate-win
 RUN mv build/win/artifactcollector32.syso artifactcollector.syso
 
 FROM golang:1.9.7

tools/resources/README.md

@@ -0,0 +1,81 @@
+# go-resources
+
+> [!IMPORTANT]  
+> You should use the `embed` package instead of this tool.
+> See https://pkg.go.dev/embed for more information.
+>
+> Only use this tool if you need to support older (< 1.16) versions of Go.
+
+go-resources is a tool to embed files into Go source code.
+
+## Installation
+
+```sh
+go install github.com/forensicanalysis/go-resources@latest
+```
+
+## Usage
+
+```sh
+resources -h
+Usage resources:
+  -output filename
+        filename to write the output to
+  -package name
+        name of the package to generate (default "main")
+  -tag tag
+        tag to use for the generated package (default no tag)
+  -trim prefix
+        path prefix to remove from the resulting file path in the virtual filesystem
+  -var name
+        name of the variable to assign the virtual filesystem to (default "FS")
+```
+
+## Optimization
+
+Generating resources result in a very high number of lines of code, 1MB
+of resources result about 5MB of code at over 87,000 lines of code. This
+is caused by the chosen representation of the file contents within the
+generated file.
+
+Instead of a (binary) string, `resources` transforms each file into an
+actual byte slice. For example, a file with content `Hello, world!` will
+be represented as follows:
+
+``` go
+var FS = map[string][]byte{
+  "/hello.txt": []byte{
+      0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64,
+      0x21,
+  },
+}
+```
+
+While this seems wasteful, the compiled binary is not really affected.
+_If you add 1MB of resources, your binary will increase 1MB as well_.
+
+However, compiling this many lines of code takes time and slows down the
+compiler. To avoid recompiling the resources every time and leverage the
+compiler cache, generate your resources into a standalone package and
+then import it, this will allow for faster iteration as you don't have
+to wait for the resources to be compiled with every change.
+
+``` sh
+mkdir -p assets
+resources -var=FS -package=assets -output=assets/assets.go your/files/here
+```
+
+``` go
+package main
+
+import "importpath/to/assets"
+
+func main() {
+  data, ok := assets.FS["your/files/here"]
+  // ...
+}
+```
+
+## Credits
+
+This is a fork of https://github.com/omeid/go-resources

tools/resources/cmd.go

@@ -0,0 +1,80 @@
+// Unfancy resources embedding with Go.
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var (
+	pkgName  = "main"
+	varName  = "FS"
+	tag      = ""
+	out      = ""
+	trimPath = ""
+)
+
+type nope struct{}
+
+func main() {
+	t0 := time.Now()
+
+	flag.StringVar(&pkgName, "package", pkgName, "`name` of the package to generate")
+	flag.StringVar(&varName, "var", varName, "`name` of the variable to assign the virtual filesystem to")
+	flag.StringVar(&tag, "tag", tag, "`tag` to use for the generated package (default no tag)")
+	flag.StringVar(&out, "output", out, "`filename` to write the output to")
+	flag.StringVar(&trimPath, "trim", trimPath, "path `prefix` to remove from the resulting file path in the virtual filesystem")
+	flag.Parse()
+
+	if out == "" {
+		flag.PrintDefaults()
+		log.Fatal("-output is required.")
+	}
+
+	config := Config{
+		Pkg: pkgName,
+		Var: varName,
+		Tag: tag,
+	}
+
+	res := New()
+	res.Config = config
+
+	files := make(map[string]nope)
+
+	for _, g := range flag.Args() {
+		matches, err := filepath.Glob(g)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, m := range matches {
+			info, err := os.Stat(m)
+
+			if !os.IsNotExist(err) && !info.IsDir() {
+				files[m] = nope{}
+			}
+		}
+	}
+
+	for path := range files {
+		name := filepath.ToSlash(path)
+		name = strings.TrimPrefix(name, trimPath)
+
+		err := res.AddFile(name, path)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if err := res.Write(out); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Finished in %v. Wrote %d resources to %s", time.Since(t0), len(files), out)
+}

tools/resources/resources.go

@@ -0,0 +1,176 @@
+// Package resources provides unfancy resources embedding with Go.
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+// File mimicks the os.File and http.File interface.
+type File interface {
+	io.Reader
+	Stat() (os.FileInfo, error)
+}
+
+// New creates a new Package.
+func New() *Package {
+	return &Package{
+		Config: Config{
+			Pkg: "resources",
+			Var: "FS",
+		},
+		Files: make(map[string]File),
+	}
+}
+
+// Config defines some details about the output file.
+type Config struct {
+	Pkg string // Pkg holds the package name
+	Var string // Var holds the variable name for the virtual filesystem
+	Tag string // Tag may hold an optional build tag, unless empty
+}
+
+// A Package describes a collection of files and how they should be transformed
+// to an output.
+type Package struct {
+	Config
+	Files map[string]File
+}
+
+// Add a file to the package at the give path.
+func (p *Package) Add(name string, file File) error {
+	name = filepath.ToSlash(name)
+	p.Files[name] = file
+
+	return nil
+}
+
+// AddFile is a helper function that adds the files from the path into the
+// package under the path file.
+func (p *Package) AddFile(name, path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	return p.Add(name, f)
+}
+
+// Build compiles the package and writes it into an io.Writer.
+func (p *Package) Build(out io.Writer) error {
+	return pkg.Execute(out, p)
+}
+
+// Write builds the package (via Build) and writes the output the file
+// given by the path argument.
+func (p *Package) Write(path string) error {
+	err := os.MkdirAll(filepath.Dir(path), 0700)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			log.Panicf("Failed to close file: %s", err)
+		}
+	}()
+
+	return p.Build(f)
+}
+
+var (
+	// Template.
+	pkg *template.Template
+
+	// BlockWidth allows to adjust the number of bytes per line in the generated file.
+	BlockWidth = 12
+)
+
+func reader(input io.Reader, indent int) (string, error) {
+	var (
+		buff      bytes.Buffer
+		strbuf    strings.Builder
+		isString  bool
+		err       error
+		curblock  = 0
+		linebreak = "\n" + strings.Repeat("\t", indent)
+	)
+
+	b := make([]byte, BlockWidth)
+	isString = true
+
+	for n, e := input.Read(b); e == nil; n, e = input.Read(b) {
+		for i := range n {
+			if isString {
+				if isGoASCII(rune(b[i])) {
+					strbuf.WriteByte(b[i])
+				} else {
+					isString = false
+				}
+			}
+
+			_, e = fmt.Fprintf(&buff, "0x%02x,", b[i])
+			if e != nil {
+				err = e
+
+				break
+			}
+
+			curblock++
+			if curblock < BlockWidth {
+				buff.WriteRune(' ')
+
+				continue
+			}
+
+			buff.WriteString(linebreak)
+
+			curblock = 0
+		}
+	}
+
+	if isString {
+		return "[]byte(`" + strbuf.String() + "`),", err
+	}
+
+	return "{" + linebreak + buff.String() + "\n" + strings.Repeat("\t", indent-1) + "},", err
+}
+
+func isGoASCII(b rune) bool {
+	if ((' ' <= b && b <= '~') || b == '\n' || b == '\t' || b == '\r') && b != '`' {
+		return true
+	}
+
+	return false
+}
+
+func init() {
+	pkg = template.Must(template.New("file").Funcs(template.FuncMap{"reader": reader}).Parse(fileTemplate))
+	pkg = template.Must(pkg.New("pkg").Parse(pkgTemplate))
+}
+
+const fileTemplate = `{{ reader . 4 }}`
+
+const pkgTemplate = `// Code generated by github.com/forensicanalysis/go-resources. DO NOT EDIT.
+
+{{ if .Tag }}// +build {{ .Tag }}
+
+{{ end }}
+package {{ .Pkg }}
+
+var {{ .Var }} = map[string][]byte{ {{range $path, $file := .Files }}
+	"/{{ $path }}": {{ template "file" $file }}{{ end }}
+}
+`