Generic lib

Dockerfile

@@ -8,13 +8,49 @@ MAINTAINER DeepFence
 RUN apk update  \
     && apk add --upgrade gcc musl-dev pkgconfig g++ make git
 
-COPY --from=vectorscan /vectorscan.tar.bz2 /
-RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
+RUN apk add --no-cache \
+    git \
+    make  \
+    build-base \
+    pkgconfig \
+    libpcap-dev \
+    libcap-dev \
+    openssl-dev \
+    file \
+    jansson-dev \
+    jansson-static \
+    bison \
+    tini \
+    su-exec
+
+RUN apk add --no-cache -t .build-deps py-setuptools \
+    openssl-libs-static \
+    jansson-dev \
+    build-base \
+    libc-dev \
+    file-dev \
+    automake \
+    autoconf \
+    libtool \
+    libcrypto3 \
+    flex \
+    git \
+    libmagic-static \
+    linux-headers
+
+RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \
+    && tar -zxf v4.3.2.tar.gz \
+    && cd yara-4.3.2 \
+    && ./bootstrap.sh \
+    && ./configure --prefix=/usr/local/yara --disable-dotnet --enable-magic --enable-cuckoo --disable-shared --enable-static\
+    && make \
+    && make install \
+    && cd /usr/local/ \
+    && tar -czf yara.tar.gz yara
 
 WORKDIR /home/deepfence/src/SecretScanner
 COPY . .
-RUN make clean
-RUN make
+RUN make clean && make all
 
 FROM alpine:3.18
 MAINTAINER DeepFence
@@ -30,7 +66,7 @@ RUN apk add --no-cache --upgrade tar libstdc++ libgcc docker skopeo bash podman
 RUN <<EOF
 set -eux
 
-apk update && apk add --no-cache --upgrade curl 
+apk update && apk add --no-cache --upgrade curl
 
 NERDCTL_VERSION=1.4.0
 curl -fsSLO https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-${TARGETARCH}.tar.gz

Makefile

@@ -6,10 +6,12 @@ bootstrap:
 clean:
 	-rm ./SecretScanner
 
-SecretScanner: $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
+vendor: go.mod
 	go mod tidy -v
 	go mod vendor
-	go build -ldflags="-extldflags=-static" -buildvcs=false -v .
+
+SecretScanner: vendor $(PWD)/**/*.go $(PWD)/agent-plugins-grpc/**/*.go
+	CGO_LDFLAGS="-ljansson -lcrypto -lmagic" PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) go build -buildmode=pie -ldflags="-s -w -extldflags=-static" -buildvcs=false -v .
 
 .PHONY: clean bootstrap
 

config.yaml

@@ -1,10 +1,7 @@
 # Secret Scanner Configuration File
-
-blacklisted_strings: [ ] # skip matches containing any of these strings (case sensitive)
-blacklisted_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ]
-blacklisted_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ]
-exclude_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd", "lost+found", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] # use {sep} for the OS' path seperator and {name_sep} for -  (i.e. / or \)
-
+exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ]
+exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes"]
+max_file_size: 1073741824
 
 signatures:
 - part: 'extension'

core/config.go

@@ -1,22 +1,15 @@
 package core
 
 import (
-	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 
-	"gopkg.in/yaml.v3"
+	"github.com/deepfence/match-scanner/pkg/config"
 )
 
 type Config struct {
-	BlacklistedStrings           []string          `yaml:"blacklisted_strings"`
-	BlacklistedExtensions        []string          `yaml:"blacklisted_extensions"`
-	BlacklistedPaths             []string          `yaml:"blacklisted_paths"`
-	ExcludePaths                 []string          `yaml:"exclude_paths"`
-	BlacklistedEntropyExtensions []string          `yaml:"blacklisted_entropy_extensions"`
-	Signatures                   []ConfigSignature `yaml:"signatures"`
+	Signatures []ConfigSignature `yaml:"signatures"`
 }
 
 type ConfigSignature struct {
@@ -33,11 +26,6 @@ type ConfigSignature struct {
 }
 
 func (c *Config) Merge(in *Config) {
-	c.BlacklistedStrings = mergeStringSlices(c.BlacklistedStrings, in.BlacklistedStrings)
-	c.BlacklistedExtensions = mergeStringSlices(c.BlacklistedExtensions, in.BlacklistedExtensions)
-	c.BlacklistedPaths = mergeStringSlices(c.BlacklistedPaths, in.BlacklistedPaths)
-	c.BlacklistedEntropyExtensions = mergeStringSlices(c.BlacklistedEntropyExtensions, in.BlacklistedEntropyExtensions)
-
 	signatureNames := make(map[string]bool, len(c.Signatures))
 	for _, sig := range c.Signatures {
 		signatureNames[sig.Name] = true
@@ -71,79 +59,17 @@ func mergeStringSlices(old, new []string) []string {
 	return old
 }
 
-func ParseConfig(options *Options) (*Config, error) {
-	configFileDirs := options.ConfigPath.Values()
-
-	if len(configFileDirs) > 0 {
-		if *options.MergeConfigs {
-			// merge them together onto default config in order of specification
-			config, err := getDefaultConfig()
-			if err != nil {
-				return nil, err
-			}
-
-			var subConfig *Config
-			for _, dir := range configFileDirs {
-				subConfig, err = loadConfigFile(dir)
-				if err != nil {
-					return nil, err
-				}
-				config.Merge(subConfig)
-			}
-
-			return config, nil
-		} else {
-			if len(configFileDirs) > 1 {
-				return nil, fmt.Errorf("error: Multiple config paths specified, but --merge-configs is not specified")
-			}
-
-			return loadConfigFile(configFileDirs[0])
-		}
-
-	}
-
-	return getDefaultConfig()
-}
-
-// Trying to first find the configuration next to executable
-// Helps e.g. with Drone where workdir is different than shhgit dir
-func getDefaultConfig() (*Config, error) {
-	ex, err := os.Executable()
-	if err != nil {
-		return nil, fmt.Errorf("os.Executable: %w", err)
-	}
-	dir := filepath.Dir(ex)
-	config, err := loadConfigFile(dir)
-	if err != nil {
-		dir, _ = os.Getwd()
-		return loadConfigFile(dir)
-	}
-	return config, nil
-}
-
-func loadConfigFile(configPath string) (*Config, error) {
+func loadExtractorConfigFile(options *Options) (config.Config, error) {
+	configPath := *options.ConfigPath
 	fstat, err := os.Stat(configPath)
 	if err != nil {
-		return nil, err
+		return config.Config{}, err
 	}
 
-	var data []byte
 	if fstat.IsDir() {
-		data, err = os.ReadFile(path.Join(configPath, "config.yaml"))
-	} else {
-		data, err = os.ReadFile(configPath)
-	}
-	if err != nil {
-		return nil, err
+		return config.ParseConfig(filepath.Join(configPath, "config.yaml"))
 	}
-
-	config := &Config{}
-	err = yaml.Unmarshal(data, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return config, nil
+	return config.ParseConfig(configPath)
 }
 
 func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {

core/match.go

@@ -1,10 +1,8 @@
 package core
 
 import (
-	"bytes"
 	"os"
 	"path/filepath"
-	"strings"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -31,69 +29,6 @@ func NewMatchFile(path string) MatchFile {
 	}
 }
 
-// IsSkippableFile Checks if the path is blacklisted
-func IsSkippableDir(path string, baseDir string) bool {
-	hostMountPath := *session.Options.HostMountPath
-	if hostMountPath != "" {
-		baseDir = hostMountPath
-	}
-
-	for _, skippablePathIndicator := range session.Config.BlacklistedPaths {
-		if strings.HasPrefix(path, skippablePathIndicator) || strings.HasPrefix(path, filepath.Join(baseDir, skippablePathIndicator)) {
-			return true
-		}
-
-	}
-
-	for _, excludePathIndicator := range session.Config.ExcludePaths {
-		if strings.Contains(path, excludePathIndicator) || strings.Contains(path, filepath.Join(baseDir, excludePathIndicator)) {
-			return true
-		}
-
-	}
-
-	return false
-}
-
-// IsSkippableFileExtension Checks if the file extension is blacklisted
-func IsSkippableFileExtension(path string) bool {
-	extension := strings.ToLower(filepath.Ext(path))
-	for _, skippableExt := range session.Config.BlacklistedExtensions {
-		if extension == skippableExt {
-			return true
-		}
-	}
-	return false
-}
-
-// CanCheckEntropy Checks if entropy based scanning is appropriate for this file
-func (match MatchFile) CanCheckEntropy() bool {
-	if match.Filename == "id_rsa" {
-		return false
-	}
-
-	for _, skippableExt := range session.Config.BlacklistedEntropyExtensions {
-		if match.Extension == skippableExt {
-			return false
-		}
-	}
-
-	return true
-}
-
-// ContainsBlacklistedString Checks if the input contains a blacklisted string
-func ContainsBlacklistedString(input []byte) bool {
-	for _, blacklistedString := range session.Config.BlacklistedStrings {
-		blacklistedByteStr := []byte(blacklistedString)
-		if bytes.Contains(input, blacklistedByteStr) {
-			log.Debugf("Blacklisted string %s matched", blacklistedString)
-			return true
-		}
-	}
-
-	return false
-}
-
 //// GetMatchingFiles Return the list of all applicable files inside the given directory for scanning
 // func GetMatchingFiles(dir string, baseDir string) (*bytes.Buffer, *bytes.Buffer, error) {
 //	findCmd := "find " + dir