From 351c407b67e025f548401b51569304ccd9fc8939 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 1 Jul 2024 15:09:39 +0900 Subject: [PATCH 1/9] use match-scanner library --- config.yaml | 9 +- core/config.go | 19 + core/session.go | 15 +- go.mod | 9 +- go.sum | 5 +- jobs/scan.go | 35 +- main.go | 106 ++--- output/output.go | 8 + scan/process_container.go | 174 -------- scan/process_image.go | 809 -------------------------------------- scan/scanner.go | 102 +++++ signature/signatures.go | 72 ++-- 12 files changed, 239 insertions(+), 1124 deletions(-) delete mode 100644 scan/process_container.go delete mode 100644 scan/process_image.go create mode 100644 scan/scanner.go diff --git a/config.yaml b/config.yaml index 31b89e5..3a87209 100644 --- a/config.yaml +++ b/config.yaml @@ -1,10 +1,7 @@ # Secret Scanner Configuration File - -blacklisted_strings: [ ] # skip matches containing any of these strings (case sensitive) -blacklisted_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] -blacklisted_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{sep}lib{sep}containerd", "{sep}var{sep}lib{sep}containers", "{sep}var{sep}lib{sep}crio", "{sep}var{sep}run{sep}containers", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] -exclude_paths: [ "{sep}var{sep}lib{sep}docker", "{sep}var{name_sep}lib{name_sep}docker","{sep}var{sep}lib{sep}containerd", "{sep}var{name_sep}lib{name_sep}containerd", "lost+found", "{sep}bin", "{sep}boot", "{sep}dev", "{sep}lib", "{sep}lib64", "{sep}media", "{sep}proc", "{sep}run", "{sep}sbin", "{sep}usr{sep}lib", "{sep}sys", "{sep}home{sep}kubernetes" ] # use {sep} for the OS' path seperator and {name_sep} for - (i.e. / or \) - +exclude_extensions: [ ".exe", ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf", ".zip", ".tar", ".tar.gz", ".ttf", ".lock", ".pem", ".so", ".jar", ".gz" ] +exclude_paths: ["/var/lib/docker", "/var/lib/containerd", "/dev", "/proc", "/usr/lib", "/sys", "/boot", "/run", ".home/kubernetes"] +max_file_size: 1073741824 signatures: - part: 'extension' diff --git a/core/config.go b/core/config.go index ecd8de7..20aedf8 100644 --- a/core/config.go +++ b/core/config.go @@ -1,12 +1,14 @@ package core import ( + "errors" "fmt" "os" "path" "path/filepath" "regexp" + "github.com/deepfence/match-scanner/pkg/config" "gopkg.in/yaml.v3" ) @@ -146,6 +148,23 @@ func loadConfigFile(configPath string) (*Config, error) { return config, nil } +func loadExtractorConfigFile(options *Options) (config.Config, error) { + configs := options.ConfigPath.Values() + if len(configs) != 1 { + return config.Config{}, errors.New("too many config files") + } + configPath := configs[0] + fstat, err := os.Stat(configPath) + if err != nil { + return config.Config{}, err + } + + if fstat.IsDir() { + return config.ParseConfig(filepath.Join(configPath, "config,yaml")) + } + return config.ParseConfig(configPath) +} + func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { *c = Config{} type plain Config diff --git a/core/session.go b/core/session.go index 034a3ac..ae33650 100644 --- a/core/session.go +++ b/core/session.go @@ -7,15 +7,17 @@ import ( "strings" "sync" + "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) type Session struct { sync.Mutex - Version string - Options *Options - Config *Config - Context context.Context + Version string + Options *Options + Config *Config + Context context.Context + ExtractorConfig config.Config } var ( @@ -53,6 +55,11 @@ func GetSession() *Session { os.Exit(1) } + if session.ExtractorConfig, err = loadExtractorConfigFile(session.Options); err != nil { + log.Error(err) + os.Exit(1) + } + pathSeparator := string(os.PathSeparator) nameSeperator := "-" var blacklistedPaths []string diff --git a/go.mod b/go.mod index 8d6718c..8c1f6a6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/deepfence/SecretScanner -go 1.21 +go 1.21.0 + +toolchain go1.22.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc @@ -8,7 +10,7 @@ require ( github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 - github.com/deepfence/vessel v0.12.3 + github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb github.com/fatih/color v1.16.0 github.com/flier/gohs v1.2.2 github.com/olekukonko/tablewriter v0.0.5 @@ -20,7 +22,6 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect @@ -31,6 +32,7 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect + github.com/deepfence/vessel v0.12.3 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/docker v26.1.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -55,6 +57,7 @@ require ( github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/sys/signal v0.7.0 // indirect github.com/moby/sys/user v0.1.0 // indirect + github.com/nlepage/go-tarfs v1.2.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect diff --git a/go.sum b/go.sum index c1c2a44..a5ef52d 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd0 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 h1:b7PmvEUzF2b+XJ5XxZJNt+gkInw85cxryfoOfCkLL3c= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2/go.mod h1:jHS6Adf3VrxnKZZ3RY10BirtFlwWj99Zd4JBAhP9SqM= +github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb h1:E3ffVItZVnhj1CD6UO/FPKyPz7Osinc4M770Jmm4JKc= +github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= github.com/deepfence/vessel v0.12.3 h1:C34t+sV+JoFdfYhg+uyS+YOEDAFIYjBKHShD3emDISA= github.com/deepfence/vessel v0.12.3/go.mod h1:bY97YUKMm0Oxasz/9o7Te60FjWCQWUYpgiWNC1E00xo= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -133,6 +135,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/nlepage/go-tarfs v1.2.1 h1:o37+JPA+ajllGKSPfy5+YpsNHDjZnAoyfvf5GsUa+Ks= +github.com/nlepage/go-tarfs v1.2.1/go.mod h1:rno18mpMy9aEH1IiJVftFsqPyIpwqSUiAOpJYjlV2NA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -218,7 +222,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/jobs/scan.go b/jobs/scan.go index 1dad3e3..61606b4 100644 --- a/jobs/scan.go +++ b/jobs/scan.go @@ -7,11 +7,13 @@ import ( "sync" "time" + "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" pb "github.com/deepfence/agent-plugins-grpc/srcgo" + cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) @@ -45,34 +47,29 @@ func DispatchScan(r *pb.FindRequest) { close(res) }() - var secrets chan output.SecretFound + var ( + scanType scan.ScanType + nodeID string + ) if r.GetPath() != "" { - var isFirstSecret bool = true - secrets, err = scan.ScanSecretsInDirStream("", r.GetPath(), r.GetPath(), - &isFirstSecret, scanCtx) - if err != nil { - return - } + scanType = scan.DirScan + nodeID = r.GetPath() } else if r.GetImage() != nil && r.GetImage().Name != "" { - secrets, err = scan.ExtractAndScanImageStream(r.GetImage().Name, scanCtx) - if err != nil { - return - } + scanType = scan.ImageScan + nodeID = r.GetImage().Name } else if r.GetContainer() != nil && r.GetContainer().Id != "" { - secrets, err = scan.ExtractAndScanContainerStream(r.GetContainer().Id, - r.GetContainer().Namespace, scanCtx) - if err != nil { - return - } + scanType = scan.ContainerScan + nodeID = r.GetContainer().Id } else { err = fmt.Errorf("Invalid request") return } - for secret := range secrets { - writeSingleScanData(output.SecretToSecretInfo(secret), r.ScanId) - } + filters := cfg.Config2Filter(core.GetSession().ExtractorConfig) + err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) { + writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId) + }) }() } diff --git a/main.go b/main.go index e4d63e5..9f550f7 100644 --- a/main.go +++ b/main.go @@ -38,6 +38,7 @@ import ( "github.com/deepfence/SecretScanner/scan" "github.com/deepfence/SecretScanner/server" "github.com/deepfence/SecretScanner/signature" + "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) @@ -53,109 +54,51 @@ var ( // and setup the session to start scanning for secrets var session = core.GetSession() -// Scan a container image for secrets layer by layer -// @parameters -// image - Name of the container image to scan (e.g. "alpine:3.5") -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInImage(image string) (*output.JSONImageSecretsOutput, error) { - - res, err := scan.ExtractAndScanImage(image) - if err != nil { - return nil, err - } - jsonImageSecretsOutput := output.JSONImageSecretsOutput{ImageName: image} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageID(res.ImageId) - jsonImageSecretsOutput.SetSecrets(res.Secrets) - - return &jsonImageSecretsOutput, nil -} - -// Scan a directory -// @parameters -// dir - Complete path of the directory to be scanned -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInDir(dir string) (*output.JSONDirSecretsOutput, error) { - var isFirstSecret bool = true - - secrets, err := scan.ScanSecretsInDir("", "", dir, &isFirstSecret, nil) - if err != nil { - log.Error("findSecretsInDir: %s", err) - return nil, err - } - - jsonDirSecretsOutput := output.JSONDirSecretsOutput{DirName: *session.Options.Local} - jsonDirSecretsOutput.SetTime() - jsonDirSecretsOutput.SetSecrets(secrets) - - return &jsonDirSecretsOutput, nil -} - -// Scan a container for secrets -// @parameters -// containerId - Id of the container to scan (e.g. "0fdasf989i0") -// @returns -// Error, if any. Otherwise, returns nil -func findSecretsInContainer(containerId string, containerNS string) (*output.JSONImageSecretsOutput, error) { - - res, err := scan.ExtractAndScanContainer(containerId, containerNS, nil) - if err != nil { - return nil, err - } - jsonImageSecretsOutput := output.JSONImageSecretsOutput{ContainerID: containerId} - jsonImageSecretsOutput.SetTime() - jsonImageSecretsOutput.SetImageID(res.ContainerId) - jsonImageSecretsOutput.SetSecrets(res.Secrets) - - return &jsonImageSecretsOutput, nil -} - type SecretsWriter interface { WriteJSON() error WriteTable() error GetSecrets() []output.SecretFound + AddSecret(output.SecretFound) } -func runOnce(format string) { +func runOnce(filters config.Filters, format string) { var result SecretsWriter var err error node_type := "" node_id := "" + var nodeType scan.ScanType // Scan container image for secrets if len(*session.Options.ImageName) > 0 { node_type = "image" node_id = *session.Options.ImageName + nodeType = scan.ImageScan log.Infof("Scanning image %s for secrets...", *session.Options.ImageName) - result, err = findSecretsInImage(*session.Options.ImageName) - if err != nil { - log.Fatal("main: error while scanning image: %s", err) + result = &output.JSONImageSecretsOutput{ + ImageName: *session.Options.ImageName, + Secrets: []output.SecretFound{}, } - } - - // Scan local directory for secrets - if len(*session.Options.Local) > 0 { - node_id = output.GetHostname() - log.Debugf("Scanning local directory: %s", *session.Options.Local) - result, err = findSecretsInDir(*session.Options.Local) - if err != nil { - log.Fatal("main: error while scanning dir: %s", err) + } else if len(*session.Options.Local) > 0 { // Scan local directory for secrets + node_id = *session.Options.Local + nodeType = scan.DirScan + result = &output.JSONDirSecretsOutput{ + DirName: *session.Options.Local, + Secrets: []output.SecretFound{}, } - } - - // Scan existing container for secrets - if len(*session.Options.ContainerID) > 0 { + } else if len(*session.Options.ContainerID) > 0 { // Scan existing container for secrets node_type = "container_image" node_id = *session.Options.ContainerID - log.Debugf("Scanning container %s for secrets...", *session.Options.ContainerID) - result, err = findSecretsInContainer(*session.Options.ContainerID, *session.Options.ContainerNS) - if err != nil { - log.Fatal("main: error while scanning container: %s", err) + nodeType = scan.ContainerScan + result = &output.JSONImageSecretsOutput{ + ContainerID: *session.Options.ContainerID, + Secrets: []output.SecretFound{}, } } + scan.Scan(nil, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { + result.AddSecret(sf) + }) + if result == nil { log.Error("set either -local or -image-name flag") return @@ -237,6 +180,7 @@ func main() { log.Fatal("main: failed to serve: %v", err) } } else { - runOnce(*core.GetSession().Options.OutFormat) + extCfg := config.Config2Filter(core.GetSession().ExtractorConfig) + runOnce(extCfg, *core.GetSession().Options.OutFormat) } } diff --git a/output/output.go b/output/output.go index 2bd5f54..8db7250 100644 --- a/output/output.go +++ b/output/output.go @@ -73,6 +73,10 @@ func (imageOutput *JSONImageSecretsOutput) GetSecrets() []SecretFound { return imageOutput.Secrets } +func (imageOutput *JSONImageSecretsOutput) AddSecret(secret SecretFound) { + imageOutput.Secrets = append(imageOutput.Secrets, secret) +} + func (imageOutput JSONImageSecretsOutput) WriteJSON() error { return printSecretsToJSON(imageOutput) @@ -105,6 +109,10 @@ func (dirOutput JSONDirSecretsOutput) WriteTable() error { return WriteTableOutput(&dirOutput.Secrets) } +func (imageOutput *JSONDirSecretsOutput) AddSecret(secret SecretFound) { + imageOutput.Secrets = append(imageOutput.Secrets, secret) +} + func printSecretsToJSON(secretsJSON interface{}) error { file, err := json.MarshalIndent(secretsJSON, "", Indent) if err != nil { diff --git a/scan/process_container.go b/scan/process_container.go deleted file mode 100644 index e9fa2c6..0000000 --- a/scan/process_container.go +++ /dev/null @@ -1,174 +0,0 @@ -package scan - -import ( - "errors" - "os" - "strings" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - tasks "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - "github.com/deepfence/vessel" - containerdRuntime "github.com/deepfence/vessel/containerd" - crioRuntime "github.com/deepfence/vessel/crio" - dockerRuntime "github.com/deepfence/vessel/docker" - podmanRuntime "github.com/deepfence/vessel/podman" - vesselConstants "github.com/deepfence/vessel/utils" - log "github.com/sirupsen/logrus" -) - -type ContainerScan struct { - containerId string - tempDir string - namespace string - numSecrets uint -} - -// Function to retrieve contents of container -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) extractFileSystem() error { - // Auto-detect underlying container runtime - containerRuntime, endpoint, err := vessel.AutoDetectRuntime() - if err != nil { - return err - } - var containerRuntimeInterface vessel.Runtime - switch containerRuntime { - case vesselConstants.DOCKER: - containerRuntimeInterface = dockerRuntime.New(endpoint) - case vesselConstants.CONTAINERD: - containerRuntimeInterface = containerdRuntime.New(endpoint) - case vesselConstants.CRIO: - containerRuntimeInterface = crioRuntime.New(endpoint) - case vesselConstants.PODMAN: - containerRuntimeInterface = podmanRuntime.New(endpoint) - } - if containerRuntimeInterface == nil { - log.Error("Error: Could not detect container runtime") - os.Exit(1) - } - err = containerRuntimeInterface.ExtractFileSystemContainer( - containerScan.containerId, containerScan.namespace, - containerScan.tempDir+".tar") - - if err != nil { - return err - } - runCommand("mkdir", containerScan.tempDir) - _, stdErr, retVal := runCommand("tar", "-xf", containerScan.tempDir+".tar", "-C"+containerScan.tempDir) - if retVal != 0 { - return errors.New(stdErr) - } - runCommand("rm", containerScan.tempDir+".tar") - return nil -} - -// Function to scan extracted layers of container file system for secrets file by file -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) scan(scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - var isFirstSecret bool = true - - secrets, err := ScanSecretsInDir("", containerScan.tempDir, containerScan.tempDir, - &isFirstSecret, scanCtx) - if err != nil { - log.Errorf("findSecretsInContainer: %s", err) - return nil, err - } - - for _, secret := range secrets { - secret.CompleteFilename = strings.Replace(secret.CompleteFilename, containerScan.tempDir, "", 1) - } - - return secrets, nil -} - -// Function to scan extracted layers of container file system for secrets file by file -// @parameters -// containerScan - Structure with details of the container to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (containerScan *ContainerScan) scanStream(scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - var isFirstSecret bool = true - - stream, err := ScanSecretsInDirStream("", containerScan.tempDir, - containerScan.tempDir, &isFirstSecret, scanCtx) - - if err != nil { - log.Errorf("findSecretsInContainer: %s", err) - return nil, err - } - - return stream, nil -} - -type ContainerExtractionResult struct { - Secrets []output.SecretFound - ContainerId string -} - -func ExtractAndScanContainer(containerId string, namespace string, - scanCtx *tasks.ScanContext) (*ContainerExtractionResult, error) { - - tempDir, err := core.GetTmpDir(containerId) - if err != nil { - return nil, err - } - defer core.DeleteTmpDir(tempDir) - - containerScan := ContainerScan{containerId: containerId, tempDir: tempDir, namespace: namespace} - err = containerScan.extractFileSystem() - - if err != nil { - return nil, err - } - - secrets, err := containerScan.scan(scanCtx) - - if err != nil { - return nil, err - } - return &ContainerExtractionResult{ContainerId: containerScan.containerId, Secrets: secrets}, nil -} - -func ExtractAndScanContainerStream(containerId string, namespace string, - scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - tempDir, err := core.GetTmpDir(containerId) - if err != nil { - return nil, err - } - - containerScan := ContainerScan{containerId: containerId, tempDir: tempDir, namespace: namespace} - err = containerScan.extractFileSystem() - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - stream, err := containerScan.scanStream(scanCtx) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - defer core.DeleteTmpDir(tempDir) - defer close(res) - for i := range stream { - res <- i - } - }() - - return res, nil -} diff --git a/scan/process_image.go b/scan/process_image.go deleted file mode 100644 index e4c96f7..0000000 --- a/scan/process_image.go +++ /dev/null @@ -1,809 +0,0 @@ -package scan - -import ( - "archive/tar" - "bufio" - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path" - "path/filepath" - "strings" - "syscall" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - "github.com/deepfence/SecretScanner/signature" - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - "github.com/deepfence/vessel" - log "github.com/sirupsen/logrus" -) - -// Data type to store details about the container image after parsing manifest -type manifestItem struct { - Config string - RepoTags []string - Layers []string - LayerIds []string `json:",omitempty"` -} - -var ( - imageTarFileName = "save-output.tar" - maxSecretsExceeded = errors.New("number of secrets exceeded max-secrets") -) - -const ( - secret_pipeline_size = 100 -) - -type ImageScan struct { - imageName string - imageId string - tempDir string - imageManifest manifestItem - numSecrets uint -} - -// Function to retrieve contents of container images layer by layer -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) extractImage(saveImage bool) error { - imageName := imageScan.imageName - tempDir := imageScan.tempDir - imageScan.numSecrets = 0 - - if saveImage { - err := imageScan.saveImageData() - if err != nil { - log.Errorf("scanImage: Could not save container image: %s. Check if the image name is correct.", err) - return err - } - } - - _, err := extractTarFile(imageName, path.Join(tempDir, imageTarFileName), tempDir) - if err != nil { - log.Errorf("scanImage: Could not extract image tar file: %s", err) - return err - } - - imageManifest, err := extractDetailsFromManifest(tempDir) - if err != nil { - log.Errorf("ProcessImageLayers: Could not get image's history: %s,"+ - " please specify repo:tag and check disk space", err.Error()) - return err - } - - imageScan.imageManifest = imageManifest - // reading image id from imanifest file json path and tripping off extension - imageScan.imageId = strings.TrimSuffix(imageScan.imageManifest.Config, ".json") - - return nil -} - -// Function to scan extracted layers of container images for secrets file by file -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) scan(scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - tempDir := imageScan.tempDir - defer core.DeleteTmpDir(tempDir) - - tempSecretsFound, err := imageScan.processImageLayers(tempDir, scanCtx) - if err != nil { - log.Error("scanImage: %s", err) - return tempSecretsFound, err - } - - return tempSecretsFound, nil -} - -// Function to scan extracted layers of container images for secrets file by file -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors, if any. Otherwise, returns nil -func (imageScan *ImageScan) scanStream(scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - return imageScan.processImageLayersStream(imageScan.tempDir, scanCtx) -} - -func readFile(path string) ([]byte, error) { - var content string - file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm) - if err != nil { - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - content += scanner.Text() + "\n" - } - return []byte(content), nil -} - -func scanFile(filePath, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { - contents, err := readFile(filePath) - if err != nil { - return nil, err - } - // fmt.Println(relPath, file.Filename, file.Extension, layer) - secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) - if err != nil { - return nil, err - } - return secrets, nil -} - -// ScanSecretsInDir Scans a given directory recursively to find all secrets inside any file in the dir -// @parameters -// layer - layer ID, if we are scanning directory inside container image -// baseDir - Parent directory -// fullDir - Complete path of the directory to be scanned -// isFirstSecret - indicates if some secrets are already printed, used to properly format json -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func ScanSecretsInDir(layer string, baseDir string, fullDir string, - isFirstSecret *bool, scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - var secretsFound []output.SecretFound - matchedRuleSet := map[uint]uint{} - - session := core.GetSession() - - if layer != "" { - core.UpdateDirsPermissionsRW(fullDir) - } - - maxFileSize := *session.Options.MaximumFileSize * 1024 - numSecrets := uint(0) - - walkErr := filepath.WalkDir(fullDir, func(path string, f os.DirEntry, err error) error { - if err != nil { - log.Debugf("Error in filepath.Walk: %s", err) - return err - } - - err = scanCtx.Checkpoint("walking in directories") - if err != nil { - return err - } - - var scanDirPath string - if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) - if scanDirPath == "" { - scanDirPath = "/" - } - } else { - scanDirPath = path - } - - if f.IsDir() { - if core.IsSkippableDir(scanDirPath, baseDir) { - return filepath.SkipDir - } - return nil - } - - // No need to scan sym links. This avoids hangs when scanning stderr, stdour or special file descriptors - // Also, the pointed files will anyway be scanned directly - if !f.Type().IsRegular() { - return nil - } - - finfo, err := f.Info() - if err != nil { - log.Warnf("Skipping %v as info could not be retrieved: %v", path, err) - return nil - } - - if uint(finfo.Size()) > maxFileSize || core.IsSkippableFileExtension(path) { - return nil - } - - file := core.NewMatchFile(path) - - relPath, err := filepath.Rel(filepath.Join(baseDir, layer), file.Path) - if err != nil { - log.Warnf("scanSecretsInDir: Couldn't remove prefix of path: %s %s %s", - baseDir, layer, file.Path) - relPath = file.Path - } - - // Add RW permissions for reading and deleting contents of containers, not for regular file system - if layer != "" { - err = os.Chmod(file.Path, 0600) - if err != nil { - log.Errorf("scanSecretsInDir changine file permission: %s", err) - } - } - - log.Debugf("attempting scanFile on: %+v, relPath: %s", file, relPath) - - secrets, err := scanFile(file.Path, relPath, file.Filename, file.Extension, layer, &numSecrets, matchedRuleSet) - if err != nil { - log.Infof("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) - log.Errorf("scanSecretsInDir: %s", err) - } else { - if len(secrets) > 0 { - secretsFound = append(secretsFound, secrets...) - } - } - - secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - secretsFound = append(secretsFound, secrets...) - - log.Debugf("scan completed for file: %+v, numSecrets: %d", file, numSecrets) - - // Don't report secrets if number of secrets exceeds MAX value - if numSecrets >= *session.Options.MaxSecrets { - return maxSecretsExceeded - } - return nil - }) - - if walkErr != nil { - if walkErr == maxSecretsExceeded { - log.Warnf("filepath.Walk: %s", walkErr) - } else { - log.Errorf("Error in filepath.Walk: %s", walkErr) - } - } - - return secretsFound, nil -} - -// ScanSecretsInDirStream Scans a given directory recursively to find all secrets inside any file in the dir -// @parameters -// layer - layer ID, if we are scanning directory inside container image -// baseDir - Parent directory -// fullDir - Complete path of the directory to be scanned -// isFirstSecret - indicates if some secrets are already printed, used to properly format json -// @returns -// chan output.SecretFound - Channel of all secrets found -// Error - Errors if any. Otherwise, returns nil -func ScanSecretsInDirStream(layer string, baseDir string, fullDir string, - isFirstSecret *bool, scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - - res := make(chan output.SecretFound, secret_pipeline_size) - - matchedRuleSet := map[uint]uint{} - numSecrets := uint(0) - - if layer != "" { - core.UpdateDirsPermissionsRW(fullDir) - } - - go func() { - - defer close(res) - session := core.GetSession() - maxFileSize := *session.Options.MaximumFileSize * 1024 - - walkErr := filepath.WalkDir(fullDir, func(path string, f os.DirEntry, err error) error { - if err != nil { - return err - } - - err = scanCtx.Checkpoint("walking in directories") - if err != nil { - return err - } - - var scanDirPath string - if layer != "" { - scanDirPath = strings.TrimPrefix(path, baseDir+"/"+layer) - if scanDirPath == "" { - scanDirPath = "/" - } - } else { - scanDirPath = path - } - - if f.IsDir() { - if core.IsSkippableDir(scanDirPath, baseDir) { - return filepath.SkipDir - } - return nil - } - - // No need to scan sym links. This avoids hangs when scanning stderr, stdour or special file descriptors - // Also, the pointed files will anyway be scanned directly - if !f.Type().IsRegular() { - return nil - } - - finfo, err := f.Info() - if err != nil { - log.Warnf("Skipping %v as info could not be retrieved: %v", path, err) - return nil - } - - if uint(finfo.Size()) > maxFileSize || core.IsSkippableFileExtension(path) { - return nil - } - - file := core.NewMatchFile(path) - - relPath, err := filepath.Rel(filepath.Join(baseDir, layer), file.Path) - if err != nil { - log.Warnf("scanSecretsInDir: Couldn't remove prefix of path: %s %s %s", - baseDir, layer, file.Path) - relPath = file.Path - } - - // Add RW permissions for reading and deleting contents of containers, not for regular file system - if layer != "" { - err = os.Chmod(file.Path, 0600) - if err != nil { - log.Errorf("scanSecretsInDir changine file permission: %s", err) - } - } - secrets, err := scanFile(file.Path, relPath, file.Filename, file.Extension, layer, &numSecrets, matchedRuleSet) - - if err != nil { - log.Infof("relPath: %s, Filename: %s, Extension: %s, layer: %s", relPath, file.Filename, file.Extension, layer) - log.Errorf("scanSecretsInDir: %s", err) - } else { - if len(secrets) > 0 { - for i := range secrets { - res <- secrets[i] - } - } - } - - secrets = signature.MatchSimpleSignatures(relPath, file.Filename, file.Extension, layer, &numSecrets) - for i := range secrets { - res <- secrets[i] - } - // Don't report secrets if number of secrets exceeds MAX value - if numSecrets >= *session.Options.MaxSecrets { - return maxSecretsExceeded - } - return nil - }) - if walkErr != nil { - if walkErr == maxSecretsExceeded { - log.Warnf("filepath.Walk: %s", walkErr) - } else { - log.Errorf("Error in filepath.Walk: %s", walkErr) - } - } - }() - return res, nil -} - -// Extract all the layers of the container image and then find secrets in each layer one by one -// @parameters -// imageScan - Structure with details of the container image to scan -// imageManifestPath - Complete path of directory where manifest of image has been extracted -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) processImageLayers(imageManifestPath string, - scanCtx *tasks.ScanContext) ([]output.SecretFound, error) { - - var tempSecretsFound []output.SecretFound - var err error - var isFirstSecret bool = true - - // extractPath - Base directory where all the layers should be extracted to - extractPath := path.Join(imageManifestPath, core.ExtractedImageFilesDir) - layerIDs := imageScan.imageManifest.LayerIds - layerPaths := imageScan.imageManifest.Layers - - loopCntr := len(layerPaths) - var secrets []output.SecretFound - for i := 0; i < loopCntr; i++ { - log.Debugf("Analyzing layer path: %s", layerPaths[i]) - log.Debugf("Analyzing layer: %s", layerIDs[i]) - // savelayerID = layerIDs[i] - completeLayerPath := path.Join(imageManifestPath, layerPaths[i]) - targetDir := path.Join(extractPath, layerIDs[i]) - log.Debugf("Complete layer path: %s", completeLayerPath) - log.Debugf("Extracted to directory: %s", targetDir) - err = core.CreateRecursiveDir(targetDir) - if err != nil { - log.Errorf("ProcessImageLayers: Unable to create target directory to extract image layers... %s", err) - return tempSecretsFound, err - } - - _, error := extractTarFile("", completeLayerPath, targetDir) - if error != nil { - log.Errorf("ProcessImageLayers: Unable to extract image layer. Reason = %s", error.Error()) - // Don't stop. Print error and continue with remaning extracted files and other layers - // return tempSecretsFound, error - } - log.Debugf("Analyzing dir: %s", targetDir) - secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, targetDir, - &isFirstSecret, scanCtx) - - imageScan.numSecrets += uint(len(secrets)) - tempSecretsFound = append(tempSecretsFound, secrets...) - if err != nil { - log.Errorf("ProcessImageLayers: %s", err) - // return tempSecretsFound, err - } - - // Don't report secrets if number of secrets exceeds MAX value - if imageScan.numSecrets >= *core.GetSession().Options.MaxSecrets { - return tempSecretsFound, nil - } - } - - return tempSecretsFound, nil -} - -// Extract all the layers of the container image and then stream secrets in each layer one by one -// @parameters -// imageScan - Structure with details of the container image to scan -// imageManifestPath - Complete path of directory where manifest of image has been extracted -// @returns -// []output.SecretFound - List of all secrets found -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) processImageLayersStream(imageManifestPath string, - scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - var err error - var isFirstSecret bool = true - - defer close(res) - - // extractPath - Base directory where all the layers should be extracted to - extractPath := path.Join(imageManifestPath, core.ExtractedImageFilesDir) - layerIDs := imageScan.imageManifest.LayerIds - layerPaths := imageScan.imageManifest.Layers - - loopCntr := len(layerPaths) - var secrets []output.SecretFound - for i := 0; i < loopCntr; i++ { - log.Debugf("Analyzing layer path: %s", layerPaths[i]) - log.Debugf("Analyzing layer: %s", layerIDs[i]) - // savelayerID = layerIDs[i] - completeLayerPath := path.Join(imageManifestPath, layerPaths[i]) - targetDir := path.Join(extractPath, layerIDs[i]) - log.Infof("Complete layer path: %s", completeLayerPath) - log.Infof("Extracted to directory: %s", targetDir) - err = core.CreateRecursiveDir(targetDir) - if err != nil { - log.Error("ProcessImageLayers: Unable to create target directory extract image layers... %v", err) - continue - } - - _, error := extractTarFile("", completeLayerPath, targetDir) - if error != nil { - log.Errorf("ProcessImageLayers: Unable to extract image layer. Reason = %s", error.Error()) - // Don't stop. Print error and continue with remaning extracted files and other layers - continue - } - log.Debugf("Analyzing dir: %s", targetDir) - secrets, err = ScanSecretsInDir(layerIDs[i], extractPath, - targetDir, &isFirstSecret, scanCtx) - - imageScan.numSecrets += uint(len(secrets)) - for i := range secrets { - res <- secrets[i] - } - if err != nil { - log.Errorf("ProcessImageLayers: %s", err) - continue - } - - // Don't report secrets if number of secrets exceeds MAX value - if imageScan.numSecrets >= *core.GetSession().Options.MaxSecrets { - break - } - } - }() - - return res, nil -} - -// Save container image as tar file in specified directory -// @parameters -// imageScan - Structure with details of the container image to scan -// @returns -// Error - Errors if any. Otherwise, returns nil -func (imageScan *ImageScan) saveImageData() error { - imageName := imageScan.imageName - outputParam := path.Join(imageScan.tempDir, imageTarFileName) - drun, err := vessel.NewRuntime() - if err != nil { - return err - } - log.Infof("Scanning image %s for secrets...", outputParam) - _, err = drun.Save(imageName, outputParam) - - if err != nil { - return err - } - log.Infof("Image %s saved in %s", imageName, imageScan.tempDir) - return nil -} - -// Extract the contents of container image and save it in specified dir -// @parameters -// imageName - Name of the container image to save -// imageTarPath - Complete path where tarball of the image is stored -// extractPath - Complete path of directory where contents of image are to be extracted -// @returns -// string - directory where contents of image are extracted -// Error - Errors, if any. Otherwise, returns nil -func extractTarFile(imageName, imageTarPath string, extractPath string) (string, error) { - log.Debugf("Started extracting tar file %s", imageTarPath) - - path := extractPath - - // Extract the contents of image from tar file - if err := untar(imageTarPath, path); err != nil { - log.Error(err) - return "", err - } - - log.Debug("Finished extracting tar file %s", imageTarPath) - return path, nil -} - -// Extract all the details from image manifest -// @parameters -// path - Complete path where image contents are extracted -// @returns -// manifestItem - The manifestItem containing details about image layers -// Error - Errors, if any. Otherwise, returns nil -func untar(tarName string, xpath string) (err error) { - tarFile, err := os.Open(tarName) - if err != nil { - return err - } - defer func() { - err = tarFile.Close() - }() - - absPath, err := filepath.Abs(xpath) - if err != nil { - return err - } - - tr := tar.NewReader(tarFile) - if strings.HasSuffix(tarName, ".gz") || strings.HasSuffix(tarName, ".gzip") { - gz, err := gzip.NewReader(tarFile) - if err != nil { - return err - } - defer gz.Close() - tr = tar.NewReader(gz) - } - - // untar each segment - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - - // determine proper file path info - finfo := hdr.FileInfo() - fileName := hdr.Name - if filepath.IsAbs(fileName) { - fileName, err = filepath.Rel("/", fileName) - if err != nil { - return err - } - } - - absFileName := filepath.Join(absPath, fileName) - if strings.Contains(fileName, "/") { - relPath := strings.Split(fileName, "/") - var absDirPath string - if len(relPath) > 1 { - dirs := relPath[0 : len(relPath)-1] - absDirPath = filepath.Join(absPath, strings.Join(dirs, "/")) - } - if err := os.MkdirAll(absDirPath, 0755); err != nil { - log.Error(err) - } - } - - if finfo.Mode().IsDir() { - if err := os.MkdirAll(absFileName, 0755); err != nil { - return err - } - continue - } - - // create new file with original file mode - file, err := os.OpenFile(absFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, finfo.Mode().Perm()) - if err != nil { - log.Error(err) - return err - } - // fmt.Printf("x %s\n", absFileName) - n, cpErr := io.Copy(file, tr) - if closeErr := file.Close(); closeErr != nil { // close file immediately - log.Error("closeErr:" + closeErr.Error()) - return err - } - if cpErr != nil { - log.Error("copyErr:" + cpErr.Error()) - return cpErr - } - if n != finfo.Size() { - return fmt.Errorf("unexpected bytes written: wrote %d, want %d", n, finfo.Size()) - } - } - return nil -} - -// Extract all the details from image manifest -// @parameters -// path - Complete path where image contents are extracted -// @returns -// manifestItem - The manifestItem containing details about image layers -// Error - Errors, if any. Otherwise, returns nil -func extractDetailsFromManifest(path string) (manifestItem, error) { - mf, err := os.Open(path + "/manifest.json") - if err != nil { - return manifestItem{}, err - } - defer mf.Close() - - var manifest []manifestItem - if err = json.NewDecoder(mf).Decode(&manifest); err != nil { - return manifestItem{}, err - } else if len(manifest) != 1 { - return manifestItem{}, err - } - var layerIds []string - for _, layer := range manifest[0].Layers { - trimmedLayerId := strings.TrimSuffix(layer, "/layer.tar") - // manifests saved by some versions of skopeo has .tar extentions - trimmedLayerId = strings.TrimSuffix(trimmedLayerId, ".tar") - layerIds = append(layerIds, trimmedLayerId) - } - manifest[0].LayerIds = layerIds - // ImageScan.imageManifest = manifest[0] - return manifest[0], nil -} - -// Execute the specified command and return the output -// @parameters -// name - Command to be executed -// args - all the arguments to be passed to the command -// @returns -// string - contents of standard output -// string - contents of standard error -// int - exit code of the executed command -func runCommand(name string, args ...string) (stdout string, stderr string, exitCode int) { - var defaultFailedCode = 1 - var outbuf, errbuf bytes.Buffer - cmd := exec.Command(name, args...) - cmd.Stdout = &outbuf - cmd.Stderr = &errbuf - - err := cmd.Run() - stdout = outbuf.String() - stderr = errbuf.String() - - if err != nil { - // try to get the exit code - if exitError, ok := err.(*exec.ExitError); ok { - ws := exitError.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } else { - // This will happen (in OSX) if `name` is not available in $PATH, - // in this situation, exit code could not be get, and stderr will be - // empty string very likely, so we use the default fail code, and format err - // to string and set to stderr - log.Printf("Could not get exit code for failed program: %v, %v", name, args) - exitCode = defaultFailedCode - if stderr == "" { - stderr = err.Error() - } - } - } else { - // success, exitCode should be 0 if go is ok - ws := cmd.ProcessState.Sys().(syscall.WaitStatus) - exitCode = ws.ExitStatus() - } - return -} - -type ImageExtractionResult struct { - Secrets []output.SecretFound - ImageId string -} - -func ExtractAndScanImage(image string) (*ImageExtractionResult, error) { - tempDir, err := core.GetTmpDir(image) - if err != nil { - return nil, err - } - // defer core.DeleteTmpDir(tempDir) - - imageScan := ImageScan{imageName: image, imageId: "", tempDir: tempDir} - err = imageScan.extractImage(true) - - if err != nil { - return nil, err - } - - secrets, err := imageScan.scan(nil) - - if err != nil { - return nil, err - } - return &ImageExtractionResult{ImageId: imageScan.imageId, Secrets: secrets}, nil -} - -func ExtractAndScanImageStream(image string, scanCtx *tasks.ScanContext) (chan output.SecretFound, error) { - tempDir, err := core.GetTmpDir(image) - if err != nil { - return nil, err - } - - imageScan := ImageScan{imageName: image, imageId: "", tempDir: tempDir} - err = imageScan.extractImage(true) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - stream, err := imageScan.scanStream(scanCtx) - - if err != nil { - core.DeleteTmpDir(tempDir) - return nil, err - } - - res := make(chan output.SecretFound, secret_pipeline_size) - - go func() { - defer core.DeleteTmpDir(tempDir) - defer close(res) - for i := range stream { - res <- i - } - }() - - return res, nil - -} - -func ExtractAndScanFromTar(tarFolder string, imageName string, - scanCtx *tasks.ScanContext) (*ImageExtractionResult, error) { - // defer core.DeleteTmpDir(tarFolder) - - imageScan := ImageScan{imageName: imageName, imageId: "", tempDir: tarFolder} - err := imageScan.extractImage(false) - - if err != nil { - return nil, err - } - - secrets, err := imageScan.scan(scanCtx) - - if err != nil { - return nil, err - } - return &ImageExtractionResult{ImageId: imageScan.imageId, Secrets: secrets}, nil -} diff --git a/scan/scanner.go b/scan/scanner.go new file mode 100644 index 0000000..dedde88 --- /dev/null +++ b/scan/scanner.go @@ -0,0 +1,102 @@ +package scan + +import ( + "context" + "fmt" + "io" + "path/filepath" + + "github.com/deepfence/SecretScanner/output" + "github.com/deepfence/SecretScanner/signature" + tasks "github.com/deepfence/golang_deepfence_sdk/utils/tasks" + "github.com/deepfence/match-scanner/pkg/extractor" + genscan "github.com/deepfence/match-scanner/pkg/scanner" + + cfg "github.com/deepfence/match-scanner/pkg/config" + "github.com/sirupsen/logrus" +) + +type ScanType int + +const ( + DirScan ScanType = iota + ImageScan + ContainerScan +) + +func ScanTypeString(st ScanType) string { + switch st { + case DirScan: + return "host" + case ImageScan: + return "image" + case ContainerScan: + return "container" + } + return "" +} + +func scanFile(contents io.ReadSeeker, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { + + secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) + if err != nil { + return nil, err + } + return secrets, nil +} + +func Scan(ctx *tasks.ScanContext, + stype ScanType, + filters cfg.Filters, + namespace, id, scanID string, + outputFn func(output.SecretFound, string)) error { + var ( + extract extractor.FileExtractor + err error + ) + switch stype { + case DirScan: + extract, err = extractor.NewDirectoryExtractor(filters, id, true) + case ImageScan: + extract, err = extractor.NewImageExtractor(filters, namespace, id) + case ContainerScan: + extract, err = extractor.NewContainerExtractor(filters, namespace, id) + default: + err = fmt.Errorf("invalid request") + } + if err != nil { + return err + } + defer extract.Close() + + // results has to be 1 element max + // to avoid overwriting the buffer entries + results := make(chan []output.SecretFound) + defer close(results) + + go func() { + for malwares := range results { + for _, malware := range malwares { + outputFn(malware, scanID) + } + } + }() + + genscan.ApplyScan(context.Background(), extract, func(f extractor.ExtractedFile) { + if ctx != nil { + err := ctx.Checkpoint("scan_phase") + if err != nil { + return + } + } + matchedRuleSet := map[uint]uint{} + var count uint + m, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "", &count, matchedRuleSet) + if err != nil { + logrus.Infof("file: %v, err: %v", f.Filename, err) + } + + results <- m + }) + return nil +} diff --git a/signature/signatures.go b/signature/signatures.go index 472c1c8..7953d12 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -4,10 +4,13 @@ import ( // "regexp" // "regexp/syntax" // "strings" + "bufio" "bytes" "errors" + "io" "math" "regexp" + "strings" "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" @@ -44,6 +47,7 @@ var ( simpleSignatureMap map[string][]core.ConfigSignature patternSignatureMap map[string][]core.ConfigSignature hyperscanBlockDbMap map[string]hyperscan.BlockDatabase + regexpMap map[string]*regexp.Regexp signatureIDMap map[int]core.ConfigSignature ) @@ -53,6 +57,7 @@ func init() { simpleSignatureMap = make(map[string][]core.ConfigSignature) patternSignatureMap = make(map[string][]core.ConfigSignature) hyperscanBlockDbMap = make(map[string]hyperscan.BlockDatabase) + regexpMap = make(map[string]*regexp.Regexp) signatureIDMap = make(map[int]core.ConfigSignature) } @@ -99,50 +104,63 @@ func MatchSimpleSignatures(path string, filename string, extension string, layer // @returns // []output.SecretFound - List of all secrets found // Error - Errors if any. Otherwise, returns nil -func MatchPatternSignatures(contents []byte, path string, filename string, extension string, layerID string, +func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string, extension string, layerID string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { var tempSecretsFound []output.SecretFound - var hsIOData HsInputOutputData + //var hsIOData HsInputOutputData var matchingPart string - var matchingStr []byte + var matchingStr io.RuneReader for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { switch part { case FilenamePart: matchingPart = part - matchingStr = []byte(filename) + matchingStr = bufio.NewReader(strings.NewReader(filename)) case PathPart: matchingPart = part - matchingStr = []byte(path) + matchingStr = bufio.NewReader(strings.NewReader(path)) case ExtPart: matchingPart = part - matchingStr = []byte(extension) + matchingStr = bufio.NewReader(strings.NewReader(extension)) case ContentsPart: matchingPart = part - matchingStr = contents + matchingStr = bufio.NewReader(contents) } - // Ignore if string to match is empty, otherwise hyperscan can return errors - if len(matchingStr) == 0 { - continue - } - - hsIOData = HsInputOutputData{ - inputData: matchingStr, - inputDataLowerCase: bytes.ToLower(matchingStr), - completeFilename: path, - layerID: layerID, - secretsFound: &tempSecretsFound, - numSecrets: numSecrets, - matchedRuleSet: matchedRuleSet, - } - err := RunHyperscan(hyperscanBlockDbMap[matchingPart], hsIOData) - if err != nil { - log.Infof("part: %s, path: %s, filename: %s, extenstion: %s, layerID: %s", - part, path, filename, extension, layerID) - log.Warnf("MatchPatternSignatures: %s", err) - return tempSecretsFound, err + //hsIOData = HsInputOutputData{ + // inputData: matchingStr, + // inputDataLowerCase: bytes.ToLower(matchingStr), + // completeFilename: path, + // layerID: layerID, + // secretsFound: &tempSecretsFound, + // numSecrets: numSecrets, + // matchedRuleSet: matchedRuleSet, + //} + indexes := regexpMap[matchingPart].FindReaderSubmatchIndex(matchingStr) + if indexes != nil { + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: 0, + RuleName: "", + PartToMatch: part, + Match: matchingPart[indexes[0]:indexes[1]], + Regex: regexpMap[matchingPart].String(), + Severity: "", + SeverityScore: 0, + PrintBufferStartIndex: 0, + MatchFromByte: 0, + MatchToByte: 0, + CompleteFilename: filename, + MatchedContents: "", + }) } + //err := RunHyperscan(hyperscanBlockDbMap[matchingPart], hsIOData) + //if err != nil { + // log.Infof("part: %s, path: %s, filename: %s, extenstion: %s, layerID: %s", + // part, path, filename, extension, layerID) + // log.Warnf("MatchPatternSignatures: %s", err) + // return tempSecretsFound, err + //} } return tempSecretsFound, nil From bad771aa2ca51c71eb87fb3b644427c00bf745c6 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 1 Jul 2024 15:44:09 +0900 Subject: [PATCH 2/9] remove hyperscan --- core/config.go | 23 +---- core/match.go | 65 ------------ core/options.go | 8 +- core/session.go | 16 --- go.mod | 1 - go.sum | 10 -- main.go | 2 +- signature/hs_pattens.go | 104 ------------------- signature/signatures.go | 217 +++++++++------------------------------- 9 files changed, 56 insertions(+), 390 deletions(-) delete mode 100644 signature/hs_pattens.go diff --git a/core/config.go b/core/config.go index 20aedf8..7bed135 100644 --- a/core/config.go +++ b/core/config.go @@ -1,7 +1,6 @@ package core import ( - "errors" "fmt" "os" "path" @@ -13,12 +12,7 @@ import ( ) type Config struct { - BlacklistedStrings []string `yaml:"blacklisted_strings"` - BlacklistedExtensions []string `yaml:"blacklisted_extensions"` - BlacklistedPaths []string `yaml:"blacklisted_paths"` - ExcludePaths []string `yaml:"exclude_paths"` - BlacklistedEntropyExtensions []string `yaml:"blacklisted_entropy_extensions"` - Signatures []ConfigSignature `yaml:"signatures"` + Signatures []ConfigSignature `yaml:"signatures"` } type ConfigSignature struct { @@ -35,11 +29,6 @@ type ConfigSignature struct { } func (c *Config) Merge(in *Config) { - c.BlacklistedStrings = mergeStringSlices(c.BlacklistedStrings, in.BlacklistedStrings) - c.BlacklistedExtensions = mergeStringSlices(c.BlacklistedExtensions, in.BlacklistedExtensions) - c.BlacklistedPaths = mergeStringSlices(c.BlacklistedPaths, in.BlacklistedPaths) - c.BlacklistedEntropyExtensions = mergeStringSlices(c.BlacklistedEntropyExtensions, in.BlacklistedEntropyExtensions) - signatureNames := make(map[string]bool, len(c.Signatures)) for _, sig := range c.Signatures { signatureNames[sig.Name] = true @@ -74,7 +63,7 @@ func mergeStringSlices(old, new []string) []string { } func ParseConfig(options *Options) (*Config, error) { - configFileDirs := options.ConfigPath.Values() + configFileDirs := options.RulesPath.Values() if len(configFileDirs) > 0 { if *options.MergeConfigs { @@ -149,18 +138,14 @@ func loadConfigFile(configPath string) (*Config, error) { } func loadExtractorConfigFile(options *Options) (config.Config, error) { - configs := options.ConfigPath.Values() - if len(configs) != 1 { - return config.Config{}, errors.New("too many config files") - } - configPath := configs[0] + configPath := *options.ConfigPath fstat, err := os.Stat(configPath) if err != nil { return config.Config{}, err } if fstat.IsDir() { - return config.ParseConfig(filepath.Join(configPath, "config,yaml")) + return config.ParseConfig(filepath.Join(configPath, "config.yaml")) } return config.ParseConfig(configPath) } diff --git a/core/match.go b/core/match.go index f48521e..c149420 100644 --- a/core/match.go +++ b/core/match.go @@ -1,10 +1,8 @@ package core import ( - "bytes" "os" "path/filepath" - "strings" log "github.com/sirupsen/logrus" ) @@ -31,69 +29,6 @@ func NewMatchFile(path string) MatchFile { } } -// IsSkippableFile Checks if the path is blacklisted -func IsSkippableDir(path string, baseDir string) bool { - hostMountPath := *session.Options.HostMountPath - if hostMountPath != "" { - baseDir = hostMountPath - } - - for _, skippablePathIndicator := range session.Config.BlacklistedPaths { - if strings.HasPrefix(path, skippablePathIndicator) || strings.HasPrefix(path, filepath.Join(baseDir, skippablePathIndicator)) { - return true - } - - } - - for _, excludePathIndicator := range session.Config.ExcludePaths { - if strings.Contains(path, excludePathIndicator) || strings.Contains(path, filepath.Join(baseDir, excludePathIndicator)) { - return true - } - - } - - return false -} - -// IsSkippableFileExtension Checks if the file extension is blacklisted -func IsSkippableFileExtension(path string) bool { - extension := strings.ToLower(filepath.Ext(path)) - for _, skippableExt := range session.Config.BlacklistedExtensions { - if extension == skippableExt { - return true - } - } - return false -} - -// CanCheckEntropy Checks if entropy based scanning is appropriate for this file -func (match MatchFile) CanCheckEntropy() bool { - if match.Filename == "id_rsa" { - return false - } - - for _, skippableExt := range session.Config.BlacklistedEntropyExtensions { - if match.Extension == skippableExt { - return false - } - } - - return true -} - -// ContainsBlacklistedString Checks if the input contains a blacklisted string -func ContainsBlacklistedString(input []byte) bool { - for _, blacklistedString := range session.Config.BlacklistedStrings { - blacklistedByteStr := []byte(blacklistedString) - if bytes.Contains(input, blacklistedByteStr) { - log.Debugf("Blacklisted string %s matched", blacklistedString) - return true - } - } - - return false -} - //// GetMatchingFiles Return the list of all applicable files inside the given directory for scanning // func GetMatchingFiles(dir string, baseDir string) (*bytes.Buffer, *bytes.Buffer, error) { // findCmd := "find " + dir diff --git a/core/options.go b/core/options.go index acc8e0c..9386aad 100644 --- a/core/options.go +++ b/core/options.go @@ -20,7 +20,8 @@ type Options struct { TempDirectory *string Local *string HostMountPath *string - ConfigPath *repeatableStringValue + ConfigPath *string + RulesPath *repeatableStringValue MergeConfigs *bool ImageName *string MultipleMatch *bool @@ -65,7 +66,8 @@ func ParseOptions() (*Options, error) { TempDirectory: flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"), Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."), HostMountPath: flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."), - ConfigPath: &repeatableStringValue{}, + ConfigPath: flag.String("config-path", "", "yaml config path"), + RulesPath: &repeatableStringValue{}, MergeConfigs: flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"), ImageName: flag.String("image-name", "", "Name of the image along with tag to scan for secrets"), MultipleMatch: flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"), @@ -84,7 +86,7 @@ func ParseOptions() (*Options, error) { FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"), FailOnLowCount: flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"), } - flag.Var(options.ConfigPath, "config-path", "Searches for config.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory. Can be specified multiple times.") + flag.Var(options.RulesPath, "rules-path", "Searches for rules.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory. Can be specified multiple times.") flag.Parse() return options, nil } diff --git a/core/session.go b/core/session.go index ae33650..250fc6d 100644 --- a/core/session.go +++ b/core/session.go @@ -4,7 +4,6 @@ import ( "context" "os" "runtime" - "strings" "sync" "github.com/deepfence/match-scanner/pkg/config" @@ -60,21 +59,6 @@ func GetSession() *Session { os.Exit(1) } - pathSeparator := string(os.PathSeparator) - nameSeperator := "-" - var blacklistedPaths []string - for _, blacklistedPath := range session.Config.BlacklistedPaths { - blacklistedPaths = append(blacklistedPaths, strings.ReplaceAll(blacklistedPath, "{sep}", pathSeparator)) - } - session.Config.BlacklistedPaths = blacklistedPaths - var excludePaths []string - for _, excludePath := range session.Config.ExcludePaths { - excludePaths = append(excludePaths, strings.ReplaceAll(excludePath, "{sep}", pathSeparator)) - excludePaths = append(excludePaths, strings.ReplaceAll(excludePath, "{name_sep}", nameSeperator)) - - } - session.Config.ExcludePaths = excludePaths - session.Start() }) diff --git a/go.mod b/go.mod index 8c1f6a6..1d91bfd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb github.com/fatih/color v1.16.0 - github.com/flier/gohs v1.2.2 github.com/olekukonko/tablewriter v0.0.5 github.com/sirupsen/logrus v1.9.3 google.golang.org/grpc v1.63.2 diff --git a/go.sum b/go.sum index a5ef52d..ef6043b 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,6 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flier/gohs v1.2.2 h1:v1Pmzvv/PgYoJhmOHadKjKr0wpudb20WcF1ZF0miiM8= -github.com/flier/gohs v1.2.2/go.mod h1:YZaZuBeDNoFW94B4j+YFo7Lv3XlkwNm9vsOvk0E3kgY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -96,8 +94,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= -github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -106,8 +102,6 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= @@ -156,10 +150,6 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= -github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= -github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= -github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/main.go b/main.go index 9f550f7..9acd604 100644 --- a/main.go +++ b/main.go @@ -166,7 +166,7 @@ func main() { signature.ProcessSignatures(session.Config.Signatures) // Build Hyperscan database for fast scanning - signature.BuildHsDb() + signature.BuildRegexes() flag.Parse() diff --git a/signature/hs_pattens.go b/signature/hs_pattens.go deleted file mode 100644 index e4e228d..0000000 --- a/signature/hs_pattens.go +++ /dev/null @@ -1,104 +0,0 @@ -package signature - -import ( - "fmt" - "os" - - "github.com/deepfence/SecretScanner/core" - "github.com/flier/gohs/hyperscan" - log "github.com/sirupsen/logrus" -) - -// Build hyperscan Databases for matching different parts in the beginning -// This can be used for repeated scanning -func BuildHsDb() { - for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { - log.Debugf("Creating hyperscan database for %s", part) - hspatterns, err := CreateHsPatterns(part) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: Unable to create patterns \"%s\": %s\n", part, err.Error()) - os.Exit(1) - } - hyperscanBlockDbMap[part] = CreateHsDb(hspatterns) - } -} - -// Create a list of hyperscan patterns with appropriate flags -// @parameters -// part - part for which list of patterns to be created: content, path, filename or extension -// @returns -// []*hyperscan.Pattern - List of hyperscan patterns -// error - Errors if any. Otherwise, returns nil -func CreateHsPatterns(part string) ([]*hyperscan.Pattern, error) { - var hsPatterns []*hyperscan.Pattern - - log.Debugf("Number of Complex Patterns for matching %s: %d", part, len(patternSignatureMap[part])) - for _, signature := range patternSignatureMap[part] { - log.Debugf("Pattern Signature %s %s %s %s %s %s %d", signature.Name, signature.Part, signature.Match, signature.Regex, signature.RegexType, signature.Severity, signature.ID) - - // Disable SomLeftMost option for large regex to avoid HS compilation failures. - // Postprocess later to find patterns - hsPattern := hyperscan.NewPattern(signature.Regex, hyperscan.DotAll|hyperscan.SomLeftMost) // hyperscan.SingleMatch - if signature.RegexType == LargeRegexType { - hsPattern = hyperscan.NewPattern(signature.Regex, hyperscan.DotAll) - if *core.GetSession().Options.MultipleMatch == false { - hsPattern = hyperscan.NewPattern(signature.Regex, hyperscan.DotAll|hyperscan.SingleMatch) - } else { - hsPattern = hyperscan.NewPattern(signature.Regex, hyperscan.DotAll) - } - } - hsPattern.Id = signature.ID - hsPatterns = append(hsPatterns, hsPattern) - } - return hsPatterns, nil -} - -// Create Hyperscan databased, which can be used for repeated scanning -// @parameters -// hsPatterns - List of hyperscan patterns -// @returns -// BlockDatabase - Hyperscan database for the given list of patterns -func CreateHsDb(hsPatterns []*hyperscan.Pattern) hyperscan.BlockDatabase { - hyperscanBlockDb, err := hyperscan.NewBlockDatabase(hsPatterns...) - if err != nil { - log.Error("ERROR: Unable to compile pattern ", err.Error()) - os.Exit(1) - } - return hyperscanBlockDb -} - -// Run hyperscan matching on the specified content -// @parameters -// hyperscanBlockDb - Hyperscan database of a list of patterns -// hsIOData - Metadata containing the contents being matched, filename, layerID etc. -// @returns -// Error - Errors if any. Otherwise, returns nil -func RunHyperscan(hyperscanBlockDb hyperscan.BlockDatabase, hsIOData HsInputOutputData) error { - hyperscanScratch, err := hyperscan.NewScratch(hyperscanBlockDb) - if err != nil { - return err - } - defer hyperscanScratch.Free() - - metadata := hsIOData - if err := hyperscanBlockDb.Scan([]byte(metadata.inputData), hyperscanScratch, hyperscanEventHandler, metadata); err != nil { - log.Infof("First 100 bytes of inputData: %s", metadata.inputData[:Min(len(metadata.inputData), 100)]) - log.Warnf("RunHyperscan: %s", err) - return err - } - return nil -} - -// This is the function that will be called by hyperscan for each match that occurs. -// @parameters -// id - ID of matched rule -// from - Start index of the match -// to - End endex of the match -// flags - This is provided by hyperscan for future use and is unused at present. -// context - Metadata containing the contents being matched, filename, layerID etc. -// @returns -// error - Errors if any. Otherwise, returns nil -func hyperscanEventHandler(id uint, from, to uint64, flags uint, context interface{}) error { - err := processHsRegexMatch(id, from, to, flags, context) - return err -} diff --git a/signature/signatures.go b/signature/signatures.go index 7953d12..e70dcc8 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -14,8 +14,7 @@ import ( "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" - "github.com/fatih/color" - "github.com/flier/gohs/hyperscan" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -30,7 +29,6 @@ const ( MaxSecretLength = 1000 // Maximum length of secret to search to find exact position of secrets in large regex patterns ) -// Data structure for passing inputs and getting outputs for hyperscan type HsInputOutputData struct { inputData []byte // Avoids extra memory during blacklist comparison, reduces memory pressure @@ -46,8 +44,6 @@ type HsInputOutputData struct { var ( simpleSignatureMap map[string][]core.ConfigSignature patternSignatureMap map[string][]core.ConfigSignature - hyperscanBlockDbMap map[string]hyperscan.BlockDatabase - regexpMap map[string]*regexp.Regexp signatureIDMap map[int]core.ConfigSignature ) @@ -56,44 +52,9 @@ func init() { // log.Infof("Initializing Patterns....") simpleSignatureMap = make(map[string][]core.ConfigSignature) patternSignatureMap = make(map[string][]core.ConfigSignature) - hyperscanBlockDbMap = make(map[string]hyperscan.BlockDatabase) - regexpMap = make(map[string]*regexp.Regexp) signatureIDMap = make(map[int]core.ConfigSignature) } -// Scan to find simple pattern matches for the path, filename and extension of this file -// @parameters -// path - Complete path of the file -// filename - Name of the file -// extension - Extension of the file -// layerID - layer ID of this file in the container image -// @returns -// []output.SecretFound - List of all secrets found -func MatchSimpleSignatures(path string, filename string, extension string, layerID string, numSecrets *uint) []output.SecretFound { - var tempSecretsFound []output.SecretFound - var matchingPart string - var matchingStr string - - for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { - switch part { - case FilenamePart: - matchingPart = FilenamePart - matchingStr = filename - case PathPart: - matchingPart = PathPart - matchingStr = path - case ExtPart: - matchingPart = ExtPart - matchingStr = extension - } - - secrets := matchString(matchingPart, matchingStr, path, layerID, numSecrets) - tempSecretsFound = append(tempSecretsFound, secrets...) - } - - return tempSecretsFound -} - // Scan to find complex pattern matches for the contents, path, filename and extension of this file // @parameters // contents - content of the file @@ -136,31 +97,32 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string // numSecrets: numSecrets, // matchedRuleSet: matchedRuleSet, //} - indexes := regexpMap[matchingPart].FindReaderSubmatchIndex(matchingStr) - if indexes != nil { - tempSecretsFound = append(tempSecretsFound, output.SecretFound{ - LayerID: layerID, - RuleID: 0, - RuleName: "", - PartToMatch: part, - Match: matchingPart[indexes[0]:indexes[1]], - Regex: regexpMap[matchingPart].String(), - Severity: "", - SeverityScore: 0, - PrintBufferStartIndex: 0, - MatchFromByte: 0, - MatchToByte: 0, - CompleteFilename: filename, - MatchedContents: "", - }) + for _, regex := range patternSignatureMap[matchingPart] { + indexes := regex.CompiledRegex.FindReaderSubmatchIndex(matchingStr) + if indexes != nil { + match := make([]byte, indexes[1]-indexes[0]) + contents.Seek(int64(indexes[0]), io.SeekStart) + _, err := contents.Read(match) + if err != nil { + logrus.Infof("content read: %v", err) + } + + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: regex.ID, + RuleName: regex.Name, + PartToMatch: part, + Match: string(match), + Regex: regex.Regex, + Severity: regex.Severity, + SeverityScore: regex.SeverityScore, + MatchFromByte: indexes[0], + MatchToByte: indexes[1], + CompleteFilename: filename, + }) + break + } } - //err := RunHyperscan(hyperscanBlockDbMap[matchingPart], hsIOData) - //if err != nil { - // log.Infof("part: %s, path: %s, filename: %s, extenstion: %s, layerID: %s", - // part, path, filename, extension, layerID) - // log.Warnf("MatchPatternSignatures: %s", err) - // return tempSecretsFound, err - //} } return tempSecretsFound, nil @@ -259,113 +221,6 @@ func addToSignatures(signature core.ConfigSignature, Signatures *[]core.ConfigSi *Signatures = append(*Signatures, signature) } -// Match simple pattern signatures with path, filename or extension -// @parameters -// part - which part to be matched: path, filename or extension -// input - input to be matched -// completeFilename - Complete path of the file -// layerID - layer ID of this file in the container image -// @returns -// []output.SecretFound - List of all secrets found -func matchString(part string, input string, completeFilename string, layerID string, - numSecrets *uint) []output.SecretFound { - var tempSecretsFound []output.SecretFound - - for _, signature := range simpleSignatureMap[part] { - // Don't report secrets if number of secrets exceeds MAX value - if *numSecrets >= *core.GetSession().Options.MaxSecrets { - log.Debugf("MAX secrets exceeded: %d", *numSecrets) - return tempSecretsFound - } - - if signature.Match == input { - if core.ContainsBlacklistedString([]byte(input)) { - log.Debugf("matchString: Skipping matches containing blacklisted strings") - continue - } - log.Debugf("Simple Signature %s %s %s %s %s %d\n", signature.Name, signature.Part, - signature.Match, signature.Regex, signature.Severity, signature.ID) - log.Debugf("Sensitive file %s found with matching %s of %s\n", - completeFilename, part, color.RedString(input)) - - secret := output.SecretFound{ - LayerID: layerID, - RuleID: signature.ID, RuleName: signature.Name, - PartToMatch: signature.Part, Match: signature.Match, Regex: signature.Regex, - Severity: signature.Severity, SeverityScore: signature.SeverityScore, - CompleteFilename: completeFilename, - MatchFromByte: 0, - MatchToByte: len(input), - MatchedContents: input, - } - tempSecretsFound = append(tempSecretsFound, secret) - *numSecrets = *numSecrets + 1 - } - } - - return tempSecretsFound -} - -// Post process after hyperscan finds signature match -// For large pattern matches, find the start of the match (SOM) before printing -// @parameters -// id - ID of matched rule -// from - Start index of the match -// to - End endex of the match -// flags - This is provided by hyperscan for future use and is unused at present. -// context - Metadata containing contents being matched, filename, layerID etc. -// @returns -// error - Errors if any. Otherwise, returns nil -func processHsRegexMatch(id uint, from, to uint64, flags uint, context interface{}) error { - var start int - hsIOData := context.(HsInputOutputData) - secrets := hsIOData.secretsFound - - // Don't report secrets if number of secrets exceeds MAX value - if *hsIOData.numSecrets >= *core.GetSession().Options.MaxSecrets { - log.Debugf("MAX secrets exceeded: %d", *hsIOData.numSecrets) - return nil - } - - sid := int(id) - start = int(from) - if signatureIDMap[sid].RegexType == LargeRegexType { - // Post process to find start of matching for large patterns - start = getStartOfLargeRegexMatch(sid, int(from), int(to), hsIOData) - } - - ito := int(to) - log.Debugf("processHsRegexMatch: %d %d %d\n", start, ito, len(hsIOData.inputData)) - if core.ContainsBlacklistedString(hsIOData.inputDataLowerCase[start:ito]) { - log.Debugf("processHsRegexMatch: Skipping matches containing blacklisted strings") - return nil - } - - // Match only once for now, later report only supersets - // Report multiple matches, only if MultipleMatch is set to true - _, exists := hsIOData.matchedRuleSet[id] // Check, if this pattern matched for this file earlier - if !exists { - hsIOData.matchedRuleSet[id] = 1 // Add to matched rules for first match - } else if *core.GetSession().Options.MultipleMatch == false { - return nil // Don't output later matches of this pattern, if multi-match is false - } else if *core.GetSession().Options.MultipleMatch == true { - hsIOData.matchedRuleSet[id] = hsIOData.matchedRuleSet[id] + 1 - if hsIOData.matchedRuleSet[id] > *core.GetSession().Options.MaxMultiMatch { - return nil // Don't output later matches of this pattern, if #Mateches > MaxThreshold - } - } - - secret, err := printMatchedSignatures(sid, start, int(to), hsIOData) - if err != nil { - log.Errorf("processHsRegexMatch: %s", err) - return nil - } - *secrets = append(*secrets, secret) - *hsIOData.numSecrets = *hsIOData.numSecrets + 1 - - return nil -} - // For large regex patterns, if Hyperscan finds a match, then // find the matching indexes directly as start of match (SOM) hyperscan flag doesn't work // for large patterns. @@ -506,3 +361,23 @@ func Max(value_0, value_1 int) int { } return value_1 } + +func BuildRegexes() { + for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { + log.Debugf("Compile regexp database for %s", part) + CompileRegexpPatterns(part) + } +} + +func CompileRegexpPatterns(part string) { + log.Debugf("Number of Complex Patterns for matching %s: %d", part, len(patternSignatureMap[part])) + for i, signature := range patternSignatureMap[part] { + log.Debugf("Pattern Signature %s %s %s %s %s %s %d", + signature.Name, signature.Part, signature.Match, + signature.Regex, signature.RegexType, signature.Severity, + signature.ID) + + signature.CompiledRegex = regexp.MustCompile(signature.Regex) + patternSignatureMap[part][i] = signature + } +} From 7ff0343615a8d221e39e625ff4bfd0ab382df431 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Tue, 2 Jul 2024 16:39:15 +0900 Subject: [PATCH 3/9] add simple regex support --- main.go | 20 +-- scan/scanner.go | 29 +++-- server/grpc.go | 28 ++--- signature/signatures.go | 270 ++++++++++++++++------------------------ 4 files changed, 145 insertions(+), 202 deletions(-) diff --git a/main.go b/main.go index 9acd604..c7bd94a 100644 --- a/main.go +++ b/main.go @@ -25,9 +25,11 @@ package main // ------------------------------------------------------------------------------ import ( + "context" "flag" "fmt" "os" + "os/signal" "path" "runtime" "strconv" @@ -38,6 +40,7 @@ import ( "github.com/deepfence/SecretScanner/scan" "github.com/deepfence/SecretScanner/server" "github.com/deepfence/SecretScanner/signature" + "github.com/deepfence/golang_deepfence_sdk/utils/tasks" "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) @@ -61,7 +64,7 @@ type SecretsWriter interface { AddSecret(output.SecretFound) } -func runOnce(filters config.Filters, format string) { +func runOnce(ctx context.Context, filters config.Filters, format string) { var result SecretsWriter var err error node_type := "" @@ -95,7 +98,11 @@ func runOnce(filters config.Filters, format string) { } } - scan.Scan(nil, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { + scanCtx := tasks.ScanContext{ + Context: ctx, + } + + scan.Scan(&scanCtx, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { result.AddSecret(sf) }) @@ -165,22 +172,21 @@ func main() { // Process and store the read signatures signature.ProcessSignatures(session.Config.Signatures) - // Build Hyperscan database for fast scanning - signature.BuildRegexes() - flag.Parse() if *core.GetSession().Options.Debug { log.SetLevel(log.DebugLevel) } + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) + if *socketPath != "" { - err := server.RunServer(*socketPath, PLUGIN_NAME) + err := server.RunServer(ctx, *socketPath, PLUGIN_NAME) if err != nil { log.Fatal("main: failed to serve: %v", err) } } else { extCfg := config.Config2Filter(core.GetSession().ExtractorConfig) - runOnce(extCfg, *core.GetSession().Options.OutFormat) + runOnce(ctx, extCfg, *core.GetSession().Options.OutFormat) } } diff --git a/scan/scanner.go b/scan/scanner.go index dedde88..f89ec16 100644 --- a/scan/scanner.go +++ b/scan/scanner.go @@ -1,10 +1,10 @@ package scan import ( - "context" "fmt" "io" "path/filepath" + "sync" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/signature" @@ -36,13 +36,19 @@ func ScanTypeString(st ScanType) string { return "" } -func scanFile(contents io.ReadSeeker, relPath, fileName, fileExtension, layer string, numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { +func scanFile(contents io.ReadSeeker, relPath, fileName, fileExtension, layer string) ([]output.SecretFound, error) { - secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer, numSecrets, matchedRuleSet) + simpleSecrets, err := signature.MatchSimpleSignatures(contents, relPath, fileName, fileExtension, layer) if err != nil { return nil, err } - return secrets, nil + + secrets, err := signature.MatchPatternSignatures(contents, relPath, fileName, fileExtension, layer) + if err != nil { + return nil, err + } + + return append(simpleSecrets, secrets...), nil } func Scan(ctx *tasks.ScanContext, @@ -72,31 +78,34 @@ func Scan(ctx *tasks.ScanContext, // results has to be 1 element max // to avoid overwriting the buffer entries results := make(chan []output.SecretFound) - defer close(results) + wg := sync.WaitGroup{} + wg.Add(1) go func() { + defer wg.Done() for malwares := range results { for _, malware := range malwares { outputFn(malware, scanID) } } }() - - genscan.ApplyScan(context.Background(), extract, func(f extractor.ExtractedFile) { + genscan.ApplyScan(ctx.Context, extract, func(f extractor.ExtractedFile) { if ctx != nil { err := ctx.Checkpoint("scan_phase") if err != nil { return } } - matchedRuleSet := map[uint]uint{} - var count uint - m, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "", &count, matchedRuleSet) + m, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "") if err != nil { logrus.Infof("file: %v, err: %v", f.Filename, err) } results <- m }) + + close(results) + wg.Wait() + return nil } diff --git a/server/grpc.go b/server/grpc.go index c868f00..0cdf8bd 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -4,10 +4,7 @@ import ( "context" "fmt" "net" - "os" - "os/signal" "sync" - "syscall" "github.com/deepfence/SecretScanner/jobs" pb "github.com/deepfence/agent-plugins-grpc/srcgo" @@ -70,12 +67,7 @@ func (s *gRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.F return &pb.FindResult{}, nil } -func RunServer(socket_path string, plugin_name string) error { - - sigs := make(chan os.Signal, 1) - done := make(chan bool, 1) - - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) +func RunServer(ctx context.Context, socket_path string, plugin_name string) error { lis, err := net.Listen("unix", fmt.Sprintf("%s", socket_path)) if err != nil { @@ -83,12 +75,6 @@ func RunServer(socket_path string, plugin_name string) error { } s := grpc.NewServer() - go func() { - <-sigs - s.GracefulStop() - done <- true - }() - jobs.ScanMap = sync.Map{} impl := &gRPCServer{socket_path: socket_path, plugin_name: plugin_name} @@ -96,11 +82,15 @@ func RunServer(socket_path string, plugin_name string) error { pb.RegisterSecretScannerServer(s, impl) pb.RegisterScannersServer(s, impl) log.Infof("main: server listening at %v", lis.Addr()) - if err := s.Serve(lis); err != nil { - return err - } + go func() { + if err := s.Serve(lis); err != nil { + log.Errorf("server: %v", err) + } + }() + + <-ctx.Done() + s.GracefulStop() - <-done log.Infof("main: exiting gracefully") return nil } diff --git a/signature/signatures.go b/signature/signatures.go index e70dcc8..f658486 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -5,8 +5,6 @@ import ( // "regexp/syntax" // "strings" "bufio" - "bytes" - "errors" "io" "math" "regexp" @@ -29,30 +27,24 @@ const ( MaxSecretLength = 1000 // Maximum length of secret to search to find exact position of secrets in large regex patterns ) -type HsInputOutputData struct { - inputData []byte - // Avoids extra memory during blacklist comparison, reduces memory pressure - inputDataLowerCase []byte - completeFilename string - layerID string - secretsFound *[]output.SecretFound - numSecrets *uint - matchedRuleSet map[uint]uint // Indicates if any rules macthed in the last iteration -} - // Different map data structures to map to appropriate signatures, DBs etc. var ( - simpleSignatureMap map[string][]core.ConfigSignature + matchRegexpMap map[string]*regexp.Regexp patternSignatureMap map[string][]core.ConfigSignature signatureIDMap map[int]core.ConfigSignature + matchSignatureMap map[string]map[string]core.ConfigSignature ) // Initialize all the data structures func init() { // log.Infof("Initializing Patterns....") - simpleSignatureMap = make(map[string][]core.ConfigSignature) patternSignatureMap = make(map[string][]core.ConfigSignature) signatureIDMap = make(map[int]core.ConfigSignature) + matchRegexpMap = make(map[string]*regexp.Regexp) + matchSignatureMap = make(map[string]map[string]core.ConfigSignature) + for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { + matchSignatureMap[part] = make(map[string]core.ConfigSignature) + } } // Scan to find complex pattern matches for the contents, path, filename and extension of this file @@ -65,57 +57,44 @@ func init() { // @returns // []output.SecretFound - List of all secrets found // Error - Errors if any. Otherwise, returns nil -func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string, extension string, layerID string, - numSecrets *uint, matchedRuleSet map[uint]uint) ([]output.SecretFound, error) { +func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string, extension string, layerID string) ([]output.SecretFound, error) { var tempSecretsFound []output.SecretFound - //var hsIOData HsInputOutputData - var matchingPart string - var matchingStr io.RuneReader + var matchingStr io.ReadSeeker for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { switch part { case FilenamePart: - matchingPart = part - matchingStr = bufio.NewReader(strings.NewReader(filename)) + matchingStr = strings.NewReader(filename) case PathPart: - matchingPart = part - matchingStr = bufio.NewReader(strings.NewReader(path)) + matchingStr = strings.NewReader(path) case ExtPart: - matchingPart = part - matchingStr = bufio.NewReader(strings.NewReader(extension)) + matchingStr = strings.NewReader(extension) case ContentsPart: - matchingPart = part - matchingStr = bufio.NewReader(contents) + matchingStr = contents } - //hsIOData = HsInputOutputData{ - // inputData: matchingStr, - // inputDataLowerCase: bytes.ToLower(matchingStr), - // completeFilename: path, - // layerID: layerID, - // secretsFound: &tempSecretsFound, - // numSecrets: numSecrets, - // matchedRuleSet: matchedRuleSet, - //} - for _, regex := range patternSignatureMap[matchingPart] { - indexes := regex.CompiledRegex.FindReaderSubmatchIndex(matchingStr) + for _, signature := range patternSignatureMap[part] { + matchingStr.Seek(0, io.SeekStart) + indexes := signature.CompiledRegex.FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) if indexes != nil { match := make([]byte, indexes[1]-indexes[0]) - contents.Seek(int64(indexes[0]), io.SeekStart) - _, err := contents.Read(match) + matchingStr.Seek(int64(indexes[0]), io.SeekStart) + _, err := matchingStr.Read(match) if err != nil { logrus.Infof("content read: %v", err) } + matchStr := string(match) tempSecretsFound = append(tempSecretsFound, output.SecretFound{ LayerID: layerID, - RuleID: regex.ID, - RuleName: regex.Name, + RuleID: signature.ID, + RuleName: signature.Name, PartToMatch: part, - Match: string(match), - Regex: regex.Regex, - Severity: regex.Severity, - SeverityScore: regex.SeverityScore, + Match: matchStr, + MatchedContents: matchStr, + Regex: signature.Regex, + Severity: signature.Severity, + SeverityScore: signature.SeverityScore, MatchFromByte: indexes[0], MatchToByte: indexes[1], CompleteFilename: filename, @@ -128,15 +107,66 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string return tempSecretsFound, nil } +func MatchSimpleSignatures(contents io.ReadSeeker, path string, filename string, extension string, layerID string) ([]output.SecretFound, error) { + var tempSecretsFound []output.SecretFound + var matchingStr io.ReadSeeker + + for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { + if _, has := matchRegexpMap[part]; !has { + continue + } + + switch part { + case FilenamePart: + matchingStr = strings.NewReader(filename) + case PathPart: + matchingStr = strings.NewReader(path) + case ExtPart: + matchingStr = strings.NewReader(extension) + case ContentsPart: + matchingStr = contents + } + + indexes := matchRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) + if indexes != nil { + match := make([]byte, indexes[1]-indexes[0]) + matchingStr.Seek(int64(indexes[0]), io.SeekStart) + _, err := matchingStr.Read(match) + if err != nil { + logrus.Infof("content read: %v", err) + } + matchStr := string(match) + signature := matchSignatureMap[part][matchStr] + + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: signature.ID, + RuleName: signature.Name, + PartToMatch: part, + Match: matchStr, + MatchedContents: matchStr, + Regex: signature.Regex, + Severity: signature.Severity, + SeverityScore: signature.SeverityScore, + MatchFromByte: indexes[0], + MatchToByte: indexes[1], + CompleteFilename: filename, + }) + } + } + + return tempSecretsFound, nil +} + // Process all the extracted signatures from config file, add severity and severity scores, finally // store them in appropriate maps // @parameters // configSignatures - Extracted patterns from signature config file func ProcessSignatures(configSignatures []core.ConfigSignature) { - var simpleContentSignatures []core.ConfigSignature - var simpleExtSignatures []core.ConfigSignature - var simpleFilenameSignatures []core.ConfigSignature - var simplePathSignatures []core.ConfigSignature + var simpleContentSignatures []string + var simpleExtSignatures []string + var simpleFilenameSignatures []string + var simplePathSignatures []string var patternContentSignatures []core.ConfigSignature var patternExtSignatures []core.ConfigSignature @@ -155,15 +185,21 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { log.Debugf("Simple Signature %s %s %s %s %d", signature.Name, signature.Part, signature.Match, signature.Severity, signature.ID) + matchSignatureMap[signature.Part][signature.Match] = signature + switch signature.Part { case ContentsPart: - addToSignatures(signature, &simpleContentSignatures) + simpleContentSignatures = append(simpleContentSignatures, + strings.ReplaceAll(signature.Match, ".", `\.`)) case ExtPart: - addToSignatures(signature, &simpleExtSignatures) + simpleExtSignatures = append(simpleExtSignatures, + strings.ReplaceAll(signature.Match, ".", `\.`)) case FilenamePart: - addToSignatures(signature, &simpleFilenameSignatures) + simpleFilenameSignatures = append(simpleFilenameSignatures, + strings.ReplaceAll(signature.Match, ".", `\.`)) case PathPart: - addToSignatures(signature, &simplePathSignatures) + simplePathSignatures = append(simplePathSignatures, + strings.ReplaceAll(signature.Match, ".", `\.`)) } } else { if signature.Severity == "" { @@ -197,10 +233,18 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { } - simpleSignatureMap[ContentsPart] = simpleContentSignatures - simpleSignatureMap[ExtPart] = simpleExtSignatures - simpleSignatureMap[FilenamePart] = simpleFilenameSignatures - simpleSignatureMap[PathPart] = simplePathSignatures + if len(simpleContentSignatures) != 0 { + matchRegexpMap[ContentsPart] = regexp.MustCompile(strings.Join(simpleContentSignatures, "|")) + } + if len(simpleExtSignatures) != 0 { + matchRegexpMap[ExtPart] = regexp.MustCompile(strings.Join(simpleExtSignatures, "|")) + } + if len(simpleFilenameSignatures) != 0 { + matchRegexpMap[FilenamePart] = regexp.MustCompile(strings.Join(simpleFilenameSignatures, "|")) + } + if len(simplePathSignatures) != 0 { + matchRegexpMap[PathPart] = regexp.MustCompile(strings.Join(simplePathSignatures, "|")) + } patternSignatureMap[ContentsPart] = patternContentSignatures patternSignatureMap[ExtPart] = patternExtSignatures @@ -209,7 +253,7 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { log.Debugf("Number of Complex Patterns for matching %s: %d", part, len(patternSignatureMap[part])) - log.Debugf("Number of Simple Patterns for matching %s: %d", part, len(simpleSignatureMap[part])) + log.Debugf("Number of Simple Patterns for matching %s: %d", part, len(matchSignatureMap[part])) } } @@ -221,92 +265,6 @@ func addToSignatures(signature core.ConfigSignature, Signatures *[]core.ConfigSi *Signatures = append(*Signatures, signature) } -// For large regex patterns, if Hyperscan finds a match, then -// find the matching indexes directly as start of match (SOM) hyperscan flag doesn't work -// for large patterns. -// @parameters -// sid - ID of matched rule -// from - Start index of the match -// to - End endex of the match -// hsIOData - Metadata containing contents being matched, filename, layerID etc. -// @returns -// int - Exact start index of the large complex regex matches -func getStartOfLargeRegexMatch(sid int, from, to int, hsIOData HsInputOutputData) int { - inputData := hsIOData.inputData - // secrets := hsIOData.secretsFound - - pattern := signatureIDMap[sid].CompiledRegex - // Hyperscan doesn't give the start of the match, but it give the end of the match for complex patterns - start := Max(0, to-MaxSecretLength) - // Search between [to-MaxSecretLength, to] to find exact match of secret - end := to - allMatchedIndexes := pattern.FindAllIndex(inputData[start:end], -1) - log.Debugf("Number of matches found for large regex pattern: %d", len(allMatchedIndexes)) - for i, loc := range allMatchedIndexes { - // Currently just print the last match as we know the end of the hyperscan match - if i == len(allMatchedIndexes)-1 { - // secret := printMatchedSignatures(sid, start+loc[0], start+loc[1], hsIOData) - // *secrets = append(*secrets, secret) - return start + loc[0] - } - } - - // It shouldn't reach here. Return "from" as start index, by default - return from -} - -// Print matched secrets on standard output as well as in output files in json format etc. -// @parameters -// sid - ID of matched rule -// from - Start index of the match -// to - End endex of the match -// hsIOData - Metadata containing contents being matched, filename, layerID etc. -// @returns -// output.SecretFound - secret found -// Error - Errors if any. Otherwise, returns nil -func printMatchedSignatures(sid int, from, to int, hsIOData HsInputOutputData) (output.SecretFound, error) { - inputData := hsIOData.inputData - completeFilename := hsIOData.completeFilename - layerID := hsIOData.layerID - - updatedSeverity, updatedScore := calculateSeverity(inputData[from:to], signatureIDMap[sid].Severity, signatureIDMap[sid].SeverityScore) - - log.Debugf("Pattern Signature %s %s %s %s %s %s %.2f %d", signatureIDMap[sid].Name, signatureIDMap[sid].Part, - signatureIDMap[sid].Match, signatureIDMap[sid].Regex, signatureIDMap[sid].RegexType, - updatedSeverity, updatedScore, signatureIDMap[sid].ID) - // fmt.Println(signatureIDMap[sid].Name, signatureIDMap[sid].Part, signatureIDMap[sid].Match, signatureIDMap[sid].Regex, - // signatureIDMap[sid].RegexType, updatedSeverity, updatedScore, signatureIDMap[sid].ID) - log.Debugf("Secret found in %s of %s within bytes %d and %d", signatureIDMap[sid].Part, completeFilename, from, to) - // fmt.Println("Secret found in", signatureIDMap[sid].Part, "of", completeFilename, "withing bytes", from, "and", to) - - start := Max(0, bytes.LastIndexByte(inputData[:from], '\n')) // Avoid -ve value from IndexByte - end := to + Max(0, bytes.IndexByte(inputData[to:], '\n')) // Avoid -ve value from IndexByte - - // Display max 50 bytes before and after the maching string - start = Max(start, from-50) - end = Min(end, to+50) - - if !(0 <= start && start <= from && from <= to && to <= end && end <= len(inputData)) { - return output.SecretFound{}, errors.New("index out of bound while printing matched signatures") - } - - // coloredMatch := fmt.Sprintf("%s%s%s\n", inputData[start:from], color.RedString(string(inputData[from:to])), inputData[to:end]) - // //log.Infof("%s%s%s\n", inputData[start:from], color.RedString(string(inputData[from:to])), inputData[to:end]) - // log.Infof(coloredMatch) - - secret := output.SecretFound{ - LayerID: layerID, - RuleID: sid, RuleName: signatureIDMap[sid].Name, - PartToMatch: signatureIDMap[sid].Part, Match: signatureIDMap[sid].Match, Regex: signatureIDMap[sid].Regex, - Severity: updatedSeverity, SeverityScore: updatedScore, - CompleteFilename: completeFilename, - PrintBufferStartIndex: start, MatchFromByte: from - start, MatchToByte: to - start, - MatchedContents: string(inputData[start:end]), - } - - return secret, nil -} - // Update severity and score based on length of match // @parameters // inputMatch - Matched portion of the input @@ -361,23 +319,3 @@ func Max(value_0, value_1 int) int { } return value_1 } - -func BuildRegexes() { - for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { - log.Debugf("Compile regexp database for %s", part) - CompileRegexpPatterns(part) - } -} - -func CompileRegexpPatterns(part string) { - log.Debugf("Number of Complex Patterns for matching %s: %d", part, len(patternSignatureMap[part])) - for i, signature := range patternSignatureMap[part] { - log.Debugf("Pattern Signature %s %s %s %s %s %s %d", - signature.Name, signature.Part, signature.Match, - signature.Regex, signature.RegexType, signature.Severity, - signature.ID) - - signature.CompiledRegex = regexp.MustCompile(signature.Regex) - patternSignatureMap[part][i] = signature - } -} From 0f5a0991cc5e1f1ad45b05f78d06d84e6a6585fd Mon Sep 17 00:00:00 2001 From: Harshvardhan Karn Date: Wed, 3 Jul 2024 15:00:04 +0530 Subject: [PATCH 4/9] refactor & fix for path name (#122) --- go.mod | 4 +--- main.go | 5 +++-- scan/scanner.go | 11 ++++++----- signature/signatures.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 1d91bfd..4b05e2b 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/deepfence/SecretScanner -go 1.21.0 - -toolchain go1.22.0 +go 1.22.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc diff --git a/main.go b/main.go index c7bd94a..c4af599 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ import ( "path" "runtime" "strconv" + "sync/atomic" "time" "github.com/deepfence/SecretScanner/core" @@ -100,6 +101,7 @@ func runOnce(ctx context.Context, filters config.Filters, format string) { scanCtx := tasks.ScanContext{ Context: ctx, + IsAlive: atomic.Bool{}, } scan.Scan(&scanCtx, nodeType, filters, "", node_id, "", func(sf output.SecretFound, s string) { @@ -157,7 +159,6 @@ func runOnce(ctx context.Context, filters config.Filters, format string) { } func main() { - log.SetOutput(os.Stderr) log.SetLevel(log.InfoLevel) log.SetReportCaller(true) @@ -183,7 +184,7 @@ func main() { if *socketPath != "" { err := server.RunServer(ctx, *socketPath, PLUGIN_NAME) if err != nil { - log.Fatal("main: failed to serve: %v", err) + log.Fatalf("main: failed to serve: %v", err) } } else { extCfg := config.Config2Filter(core.GetSession().ExtractorConfig) diff --git a/scan/scanner.go b/scan/scanner.go index f89ec16..3a288f7 100644 --- a/scan/scanner.go +++ b/scan/scanner.go @@ -83,9 +83,9 @@ func Scan(ctx *tasks.ScanContext, wg.Add(1) go func() { defer wg.Done() - for malwares := range results { - for _, malware := range malwares { - outputFn(malware, scanID) + for secrets := range results { + for _, secret := range secrets { + outputFn(secret, scanID) } } }() @@ -96,12 +96,13 @@ func Scan(ctx *tasks.ScanContext, return } } - m, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "") + logrus.Infof("Scanning file: %v", f.Filename) + s, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "") if err != nil { logrus.Infof("file: %v, err: %v", f.Filename, err) } - results <- m + results <- s }) close(results) diff --git a/signature/signatures.go b/signature/signatures.go index f658486..13108e6 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -97,7 +97,7 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string SeverityScore: signature.SeverityScore, MatchFromByte: indexes[0], MatchToByte: indexes[1], - CompleteFilename: filename, + CompleteFilename: path, }) break } @@ -150,7 +150,7 @@ func MatchSimpleSignatures(contents io.ReadSeeker, path string, filename string, SeverityScore: signature.SeverityScore, MatchFromByte: indexes[0], MatchToByte: indexes[1], - CompleteFilename: filename, + CompleteFilename: path, }) } } From 2bafb1dcaadae053ef72bbf0069230c07a7827c2 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 3 Jul 2024 18:49:47 +0900 Subject: [PATCH 5/9] optimize pattern matching --- signature/signatures.go | 77 +++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/signature/signatures.go b/signature/signatures.go index 13108e6..5a168ff 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -30,6 +30,7 @@ const ( // Different map data structures to map to appropriate signatures, DBs etc. var ( matchRegexpMap map[string]*regexp.Regexp + patternRegexpMap map[string]*regexp.Regexp patternSignatureMap map[string][]core.ConfigSignature signatureIDMap map[int]core.ConfigSignature matchSignatureMap map[string]map[string]core.ConfigSignature @@ -41,6 +42,7 @@ func init() { patternSignatureMap = make(map[string][]core.ConfigSignature) signatureIDMap = make(map[int]core.ConfigSignature) matchRegexpMap = make(map[string]*regexp.Regexp) + patternRegexpMap = make(map[string]*regexp.Regexp) matchSignatureMap = make(map[string]map[string]core.ConfigSignature) for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { matchSignatureMap[part] = make(map[string]core.ConfigSignature) @@ -73,33 +75,34 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string matchingStr = contents } - for _, signature := range patternSignatureMap[part] { - matchingStr.Seek(0, io.SeekStart) - indexes := signature.CompiledRegex.FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) - if indexes != nil { - match := make([]byte, indexes[1]-indexes[0]) - matchingStr.Seek(int64(indexes[0]), io.SeekStart) - _, err := matchingStr.Read(match) - if err != nil { - logrus.Infof("content read: %v", err) + indexes := matchRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) + if indexes != nil { + match := make([]byte, indexes[1]-indexes[0]) + matchingStr.Seek(int64(indexes[0]), io.SeekStart) + _, err := matchingStr.Read(match) + if err != nil { + logrus.Infof("content read: %v", err) + } + matchStr := string(match) + + for _, signature := range patternSignatureMap[part] { + if signature.CompiledRegex.Match(match) { + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: signature.ID, + RuleName: signature.Name, + PartToMatch: part, + Match: matchStr, + MatchedContents: matchStr, + Regex: signature.Regex, + Severity: signature.Severity, + SeverityScore: signature.SeverityScore, + MatchFromByte: indexes[0], + MatchToByte: indexes[1], + CompleteFilename: path, + }) + break } - matchStr := string(match) - - tempSecretsFound = append(tempSecretsFound, output.SecretFound{ - LayerID: layerID, - RuleID: signature.ID, - RuleName: signature.Name, - PartToMatch: part, - Match: matchStr, - MatchedContents: matchStr, - Regex: signature.Regex, - Severity: signature.Severity, - SeverityScore: signature.SeverityScore, - MatchFromByte: indexes[0], - MatchToByte: indexes[1], - CompleteFilename: path, - }) - break } } } @@ -168,6 +171,11 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { var simpleFilenameSignatures []string var simplePathSignatures []string + var patternContentReg []string + var patternExtReg []string + var patternFilenameReg []string + var patternPathReg []string + var patternContentSignatures []core.ConfigSignature var patternExtSignatures []core.ConfigSignature var patternFilenameSignatures []core.ConfigSignature @@ -220,12 +228,16 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { switch signature.Part { case ContentsPart: patternContentSignatures = append(patternContentSignatures, signature) + patternContentReg = append(patternContentReg, signature.Regex) case ExtPart: patternExtSignatures = append(patternExtSignatures, signature) + patternExtReg = append(patternExtReg, signature.Regex) case FilenamePart: patternFilenameSignatures = append(patternFilenameSignatures, signature) + patternFilenameReg = append(patternFilenameReg, signature.Regex) case PathPart: patternPathSignatures = append(patternPathSignatures, signature) + patternPathReg = append(patternPathReg, signature.Regex) } } @@ -246,6 +258,19 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { matchRegexpMap[PathPart] = regexp.MustCompile(strings.Join(simplePathSignatures, "|")) } + if len(patternContentReg) != 0 { + patternRegexpMap[ContentsPart] = regexp.MustCompile(strings.Join(patternContentReg, "|")) + } + if len(patternExtReg) != 0 { + patternRegexpMap[ExtPart] = regexp.MustCompile(strings.Join(patternExtReg, "|")) + } + if len(patternFilenameReg) != 0 { + patternRegexpMap[FilenamePart] = regexp.MustCompile(strings.Join(patternFilenameReg, "|")) + } + if len(patternPathReg) != 0 { + patternRegexpMap[PathPart] = regexp.MustCompile(strings.Join(patternPathReg, "|")) + } + patternSignatureMap[ContentsPart] = patternContentSignatures patternSignatureMap[ExtPart] = patternExtSignatures patternSignatureMap[FilenamePart] = patternFilenameSignatures From 8902eb83f0a01261eb5ab5ecbd4b64999bbf7e95 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 3 Jul 2024 18:53:12 +0900 Subject: [PATCH 6/9] use 1.21 --- go.mod | 2 +- signature/signatures.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4b05e2b..9532a1b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/deepfence/SecretScanner -go 1.22.0 +go 1.21.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc diff --git a/signature/signatures.go b/signature/signatures.go index 5a168ff..12217af 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -64,6 +64,10 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string var matchingStr io.ReadSeeker for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { + if _, has := patternRegexpMap[part]; !has { + continue + } + switch part { case FilenamePart: matchingStr = strings.NewReader(filename) @@ -75,7 +79,7 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string matchingStr = contents } - indexes := matchRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) + indexes := patternRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) if indexes != nil { match := make([]byte, indexes[1]-indexes[0]) matchingStr.Seek(int64(indexes[0]), io.SeekStart) From 3b250b82b1eac410e0704156c096929ae65f791b Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 3 Jul 2024 22:09:17 +0900 Subject: [PATCH 7/9] optimize regexp match --- scan/scanner.go | 2 +- signature/signatures.go | 72 +++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 47 deletions(-) diff --git a/scan/scanner.go b/scan/scanner.go index 3a288f7..0182c47 100644 --- a/scan/scanner.go +++ b/scan/scanner.go @@ -96,7 +96,7 @@ func Scan(ctx *tasks.ScanContext, return } } - logrus.Infof("Scanning file: %v", f.Filename) + logrus.Debugf("Scanning file: %v", f.Filename) s, err := scanFile(f.Content, f.Filename, filepath.Base(f.Filename), filepath.Ext(f.Filename), "") if err != nil { logrus.Infof("file: %v, err: %v", f.Filename, err) diff --git a/signature/signatures.go b/signature/signatures.go index 12217af..5db0d47 100644 --- a/signature/signatures.go +++ b/signature/signatures.go @@ -30,7 +30,6 @@ const ( // Different map data structures to map to appropriate signatures, DBs etc. var ( matchRegexpMap map[string]*regexp.Regexp - patternRegexpMap map[string]*regexp.Regexp patternSignatureMap map[string][]core.ConfigSignature signatureIDMap map[int]core.ConfigSignature matchSignatureMap map[string]map[string]core.ConfigSignature @@ -42,7 +41,6 @@ func init() { patternSignatureMap = make(map[string][]core.ConfigSignature) signatureIDMap = make(map[int]core.ConfigSignature) matchRegexpMap = make(map[string]*regexp.Regexp) - patternRegexpMap = make(map[string]*regexp.Regexp) matchSignatureMap = make(map[string]map[string]core.ConfigSignature) for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { matchSignatureMap[part] = make(map[string]core.ConfigSignature) @@ -64,9 +62,6 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string var matchingStr io.ReadSeeker for _, part := range []string{ContentsPart, FilenamePart, PathPart, ExtPart} { - if _, has := patternRegexpMap[part]; !has { - continue - } switch part { case FilenamePart: @@ -79,34 +74,32 @@ func MatchPatternSignatures(contents io.ReadSeeker, path string, filename string matchingStr = contents } - indexes := patternRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) - if indexes != nil { - match := make([]byte, indexes[1]-indexes[0]) - matchingStr.Seek(int64(indexes[0]), io.SeekStart) - _, err := matchingStr.Read(match) - if err != nil { - logrus.Infof("content read: %v", err) - } - matchStr := string(match) - - for _, signature := range patternSignatureMap[part] { - if signature.CompiledRegex.Match(match) { - tempSecretsFound = append(tempSecretsFound, output.SecretFound{ - LayerID: layerID, - RuleID: signature.ID, - RuleName: signature.Name, - PartToMatch: part, - Match: matchStr, - MatchedContents: matchStr, - Regex: signature.Regex, - Severity: signature.Severity, - SeverityScore: signature.SeverityScore, - MatchFromByte: indexes[0], - MatchToByte: indexes[1], - CompleteFilename: path, - }) - break + for _, signature := range patternSignatureMap[part] { + matchingStr.Seek(0, io.SeekStart) + indexes := signature.CompiledRegex.FindReaderIndex(bufio.NewReader(matchingStr)) + if indexes != nil { + match := make([]byte, indexes[1]-indexes[0]) + matchingStr.Seek(int64(indexes[0]), io.SeekStart) + _, err := matchingStr.Read(match) + if err != nil { + logrus.Infof("content read: %v", err) } + matchStr := string(match) + tempSecretsFound = append(tempSecretsFound, output.SecretFound{ + LayerID: layerID, + RuleID: signature.ID, + RuleName: signature.Name, + PartToMatch: part, + Match: matchStr, + MatchedContents: matchStr, + Regex: signature.Regex, + Severity: signature.Severity, + SeverityScore: signature.SeverityScore, + MatchFromByte: indexes[0], + MatchToByte: indexes[1], + CompleteFilename: path, + }) + break } } } @@ -134,7 +127,7 @@ func MatchSimpleSignatures(contents io.ReadSeeker, path string, filename string, matchingStr = contents } - indexes := matchRegexpMap[part].FindReaderSubmatchIndex(bufio.NewReaderSize(matchingStr, 2048)) + indexes := matchRegexpMap[part].FindReaderIndex(bufio.NewReader(matchingStr)) if indexes != nil { match := make([]byte, indexes[1]-indexes[0]) matchingStr.Seek(int64(indexes[0]), io.SeekStart) @@ -262,19 +255,6 @@ func ProcessSignatures(configSignatures []core.ConfigSignature) { matchRegexpMap[PathPart] = regexp.MustCompile(strings.Join(simplePathSignatures, "|")) } - if len(patternContentReg) != 0 { - patternRegexpMap[ContentsPart] = regexp.MustCompile(strings.Join(patternContentReg, "|")) - } - if len(patternExtReg) != 0 { - patternRegexpMap[ExtPart] = regexp.MustCompile(strings.Join(patternExtReg, "|")) - } - if len(patternFilenameReg) != 0 { - patternRegexpMap[FilenamePart] = regexp.MustCompile(strings.Join(patternFilenameReg, "|")) - } - if len(patternPathReg) != 0 { - patternRegexpMap[PathPart] = regexp.MustCompile(strings.Join(patternPathReg, "|")) - } - patternSignatureMap[ContentsPart] = patternContentSignatures patternSignatureMap[ExtPart] = patternExtSignatures patternSignatureMap[FilenamePart] = patternFilenameSignatures From 2a7cd8a8373cb65ea8cbf3ebcace2559a779f23b Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 8 Jul 2024 13:05:50 +0900 Subject: [PATCH 8/9] use yarascanner --- core/config.go | 78 ----------------------------------- core/options.go | 107 +++++++++++++++++++++++++----------------------- core/session.go | 5 --- go.mod | 54 +++++++++++++----------- go.sum | 107 ++++++++++++++++++++++++++---------------------- main.go | 35 ++++++++++++---- 6 files changed, 169 insertions(+), 217 deletions(-) diff --git a/core/config.go b/core/config.go index 7bed135..acad7e1 100644 --- a/core/config.go +++ b/core/config.go @@ -1,14 +1,11 @@ package core import ( - "fmt" "os" - "path" "path/filepath" "regexp" "github.com/deepfence/match-scanner/pkg/config" - "gopkg.in/yaml.v3" ) type Config struct { @@ -62,81 +59,6 @@ func mergeStringSlices(old, new []string) []string { return old } -func ParseConfig(options *Options) (*Config, error) { - configFileDirs := options.RulesPath.Values() - - if len(configFileDirs) > 0 { - if *options.MergeConfigs { - // merge them together onto default config in order of specification - config, err := getDefaultConfig() - if err != nil { - return nil, err - } - - var subConfig *Config - for _, dir := range configFileDirs { - subConfig, err = loadConfigFile(dir) - if err != nil { - return nil, err - } - config.Merge(subConfig) - } - - return config, nil - } else { - if len(configFileDirs) > 1 { - return nil, fmt.Errorf("error: Multiple config paths specified, but --merge-configs is not specified") - } - - return loadConfigFile(configFileDirs[0]) - } - - } - - return getDefaultConfig() -} - -// Trying to first find the configuration next to executable -// Helps e.g. with Drone where workdir is different than shhgit dir -func getDefaultConfig() (*Config, error) { - ex, err := os.Executable() - if err != nil { - return nil, fmt.Errorf("os.Executable: %w", err) - } - dir := filepath.Dir(ex) - config, err := loadConfigFile(dir) - if err != nil { - dir, _ = os.Getwd() - return loadConfigFile(dir) - } - return config, nil -} - -func loadConfigFile(configPath string) (*Config, error) { - fstat, err := os.Stat(configPath) - if err != nil { - return nil, err - } - - var data []byte - if fstat.IsDir() { - data, err = os.ReadFile(path.Join(configPath, "config.yaml")) - } else { - data, err = os.ReadFile(configPath) - } - if err != nil { - return nil, err - } - - config := &Config{} - err = yaml.Unmarshal(data, config) - if err != nil { - return nil, err - } - - return config, nil -} - func loadExtractorConfigFile(options *Options) (config.Config, error) { configPath := *options.ConfigPath fstat, err := os.Stat(configPath) diff --git a/core/options.go b/core/options.go index 9386aad..5ca5273 100644 --- a/core/options.go +++ b/core/options.go @@ -14,31 +14,34 @@ const ( ) type Options struct { - Threads *int - Debug *bool - MaximumFileSize *uint - TempDirectory *string - Local *string - HostMountPath *string - ConfigPath *string - RulesPath *repeatableStringValue - MergeConfigs *bool - ImageName *string - MultipleMatch *bool - MaxMultiMatch *uint - MaxSecrets *uint - ContainerID *string - ContainerNS *string - WorkersPerScan *int - InactiveThreshold *int - OutFormat *string - ConsoleURL *string - ConsolePort *int - DeepfenceKey *string - FailOnCount *int - FailOnHighCount *int - FailOnMediumCount *int - FailOnLowCount *int + Threads *int + Debug *bool + MaximumFileSize *uint + TempDirectory *string + Local *string + HostMountPath *string + ConfigPath *string + RulesPath *string + RulesListingURL *string + FailOnCompileWarning *bool + EnableUpdater *bool + MergeConfigs *bool + ImageName *string + MultipleMatch *bool + MaxMultiMatch *uint + MaxSecrets *uint + ContainerID *string + ContainerNS *string + WorkersPerScan *int + InactiveThreshold *int + OutFormat *string + ConsoleURL *string + ConsolePort *int + DeepfenceKey *string + FailOnCount *int + FailOnHighCount *int + FailOnMediumCount *int + FailOnLowCount *int } type repeatableStringValue struct { @@ -60,33 +63,35 @@ func (v *repeatableStringValue) Values() []string { func ParseOptions() (*Options, error) { options := &Options{ - Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), - Debug: flag.Bool("debug", false, "enable debug logs"), - MaximumFileSize: flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"), - TempDirectory: flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"), - Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."), - HostMountPath: flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."), - ConfigPath: flag.String("config-path", "", "yaml config path"), - RulesPath: &repeatableStringValue{}, - MergeConfigs: flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"), - ImageName: flag.String("image-name", "", "Name of the image along with tag to scan for secrets"), - MultipleMatch: flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"), - MaxMultiMatch: flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."), - MaxSecrets: flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."), - ContainerID: flag.String("container-id", "", "Id of existing container ID"), - ContainerNS: flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"), - WorkersPerScan: flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"), - InactiveThreshold: flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"), - OutFormat: flag.String("output", TableOutput, "Output format: json or table"), - ConsoleURL: flag.String("console-url", "", "Deepfence Management Console URL"), - ConsolePort: flag.Int("console-port", 443, "Deepfence Management Console Port"), - DeepfenceKey: flag.String("deepfence-key", "", "Deepfence key for auth"), - FailOnCount: flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"), - FailOnHighCount: flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"), - FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"), - FailOnLowCount: flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"), + Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), + Debug: flag.Bool("debug", false, "enable debug logs"), + MaximumFileSize: flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"), + TempDirectory: flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"), + Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."), + HostMountPath: flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."), + ConfigPath: flag.String("config-path", "", "yaml config path"), + RulesPath: flag.String("rules-path", "", "yara rules path"), + RulesListingURL: flag.String("rules-listing-url", "", "yara rules listing url"), + FailOnCompileWarning: flag.Bool("fail-warning", false, "fail if compilation warning"), + EnableUpdater: flag.Bool("enable-updated", false, "Enable rule updater"), + MergeConfigs: flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"), + ImageName: flag.String("image-name", "", "Name of the image along with tag to scan for secrets"), + MultipleMatch: flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"), + MaxMultiMatch: flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."), + MaxSecrets: flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."), + ContainerID: flag.String("container-id", "", "Id of existing container ID"), + ContainerNS: flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"), + WorkersPerScan: flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"), + InactiveThreshold: flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"), + OutFormat: flag.String("output", TableOutput, "Output format: json or table"), + ConsoleURL: flag.String("console-url", "", "Deepfence Management Console URL"), + ConsolePort: flag.Int("console-port", 443, "Deepfence Management Console Port"), + DeepfenceKey: flag.String("deepfence-key", "", "Deepfence key for auth"), + FailOnCount: flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"), + FailOnHighCount: flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"), + FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"), + FailOnLowCount: flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"), } - flag.Var(options.RulesPath, "rules-path", "Searches for rules.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory. Can be specified multiple times.") flag.Parse() return options, nil } diff --git a/core/session.go b/core/session.go index 250fc6d..de4c5d5 100644 --- a/core/session.go +++ b/core/session.go @@ -49,11 +49,6 @@ func GetSession() *Session { os.Exit(1) } - if session.Config, err = ParseConfig(session.Options); err != nil { - log.Error(err) - os.Exit(1) - } - if session.ExtractorConfig, err = loadExtractorConfigFile(session.Options); err != nil { log.Error(err) os.Exit(1) diff --git a/go.mod b/go.mod index 9532a1b..f4262ae 100644 --- a/go.mod +++ b/go.mod @@ -4,50 +4,54 @@ go 1.21.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc +replace github.com/deepfence/YaraHunter => ../YaraHunter + require ( + github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 - github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2 - github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 - github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb + github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 + github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 + github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f github.com/fatih/color v1.16.0 github.com/olekukonko/tablewriter v0.0.5 github.com/sirupsen/logrus v1.9.3 - google.golang.org/grpc v1.63.2 - gopkg.in/yaml.v3 v3.0.1 + google.golang.org/grpc v1.64.0 ) require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Microsoft/hcsshim v0.12.3 // indirect - github.com/containerd/cgroups/v3 v3.0.2 // indirect - github.com/containerd/containerd v1.7.16 // indirect + github.com/Microsoft/hcsshim v0.12.4 // indirect + github.com/containerd/cgroups/v3 v3.0.3 // indirect + github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/ttrpc v1.2.3 // indirect + github.com/containerd/ttrpc v1.2.5 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/deepfence/vessel v0.12.3 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v26.1.1+incompatible // indirect + github.com/docker/docker v27.0.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.5 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hillu/go-yara/v4 v4.3.2 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.7.1 // indirect @@ -60,16 +64,18 @@ require ( github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect - golang.org/x/net v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.27.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect - google.golang.org/protobuf v1.34.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ef6043b..6270cf7 100644 --- a/go.sum +++ b/go.sum @@ -8,17 +8,17 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Microsoft/hcsshim v0.12.3 h1:LS9NXqXhMoqNCplK1ApmVSfB4UnVLRDWRapB6EIlxE0= -github.com/Microsoft/hcsshim v0.12.3/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= +github.com/Microsoft/hcsshim v0.12.4 h1:Ev7YUMHAHoWNm+aDSPzc5W9s6E2jyL1szpVDJeZ/Rr4= +github.com/Microsoft/hcsshim v0.12.4/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= -github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/containerd v1.7.16 h1:7Zsfe8Fkj4Wi2My6DXGQ87hiqIrmOXolm72ZEkFU5Mg= -github.com/containerd/containerd v1.7.16/go.mod h1:NL49g7A/Fui7ccmxV6zkBWwqMgmMxFWzujYCc+JLt7k= +github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0= +github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= @@ -27,25 +27,25 @@ github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/ttrpc v1.2.3 h1:4jlhbXIGvijRtNC8F/5CpuJZ7yKOBFGFOOXg1bkISz0= -github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM= +github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= +github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2 h1:yJYaf8azyF43KJiXp75IZBS7uYsQIg/FgzhNir8EMKw= -github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= -github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2 h1:b7PmvEUzF2b+XJ5XxZJNt+gkInw85cxryfoOfCkLL3c= -github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2/go.mod h1:jHS6Adf3VrxnKZZ3RY10BirtFlwWj99Zd4JBAhP9SqM= -github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb h1:E3ffVItZVnhj1CD6UO/FPKyPz7Osinc4M770Jmm4JKc= -github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= +github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y= +github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= +github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA= +github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:jHS6Adf3VrxnKZZ3RY10BirtFlwWj99Zd4JBAhP9SqM= +github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f h1:TqJx9TO/YD/W6CgDb1f4IkZqRAM31i6O0I2OnrE6xCI= +github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= github.com/deepfence/vessel v0.12.3 h1:C34t+sV+JoFdfYhg+uyS+YOEDAFIYjBKHShD3emDISA= github.com/deepfence/vessel v0.12.3/go.mod h1:bY97YUKMm0Oxasz/9o7Te60FjWCQWUYpgiWNC1E00xo= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ= -github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.2+incompatible h1:mNhCtgXNV1fIRns102grG7rdzIsGGCq1OlOD0KunZos= +github.com/docker/docker v27.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= @@ -60,9 +60,11 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= +github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -98,21 +100,24 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hillu/go-yara/v4 v4.3.2 h1:HGqUN3ORUduWZbb95RQjut4UzavGDbtt/C6SnGB3Amk= +github.com/hillu/go-yara/v4 v4.3.2/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -148,12 +153,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -164,20 +171,20 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -198,8 +205,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -217,12 +224,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -242,19 +249,19 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 h1:MTmrc2F5TZKDKXigcZetYkH04YwqtOPEQJwh4PPOgfk= -google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6/go.mod h1:2ROWwqCIx97Y7CSyp11xB8fori0wzvD6+gbacaf5c8I= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -266,8 +273,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index c4af599..c82101b 100644 --- a/main.go +++ b/main.go @@ -39,11 +39,12 @@ import ( "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/SecretScanner/server" "github.com/deepfence/SecretScanner/signature" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" + + "github.com/deepfence/YaraHunter/pkg/runner" ) const ( @@ -181,13 +182,29 @@ func main() { ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) - if *socketPath != "" { - err := server.RunServer(ctx, *socketPath, PLUGIN_NAME) - if err != nil { - log.Fatalf("main: failed to serve: %v", err) - } - } else { - extCfg := config.Config2Filter(core.GetSession().ExtractorConfig) - runOnce(ctx, extCfg, *core.GetSession().Options.OutFormat) + runnerOpts := runner.RunnerOptions{ + SocketPath: *socketPath, + RulesPath: *core.GetSession().Options.RulesPath, + RulesListingURL: *core.GetSession().Options.RulesListingURL, + HostMountPath: *core.GetSession().Options.HostMountPath, + FailOnCompileWarning: *core.GetSession().Options.FailOnCompileWarning, + Local: *core.GetSession().Options.Local, + ImageName: *core.GetSession().Options.ImageName, + ContainerID: *core.GetSession().Options.ContainerID, + ConsoleURL: *core.GetSession().Options.ConsoleURL, + ConsolePort: *core.GetSession().Options.ConsolePort, + DeepfenceKey: *core.GetSession().Options.DeepfenceKey, + OutFormat: *core.GetSession().Options.OutFormat, + FailOnHighCount: *core.GetSession().Options.FailOnHighCount, + FailOnMediumCount: *core.GetSession().Options.FailOnMediumCount, + FailOnLowCount: *core.GetSession().Options.FailOnLowCount, + FailOnCount: *core.GetSession().Options.FailOnCount, + InactiveThreshold: *core.GetSession().Options.InactiveThreshold, } + + if *core.GetSession().Options.EnableUpdater { + go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts) + } + + runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig) } From b60fa2d5f6009b7ee350f3e5f8ad2cee4eafd25a Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 8 Jul 2024 17:35:06 +0900 Subject: [PATCH 9/9] use new grpc abstraction --- Dockerfile | 46 +++++++++++++++++-- Makefile | 6 ++- go.mod | 4 +- go.sum | 2 + jobs/common.go | 4 +- jobs/scan.go | 64 +------------------------- jobs/status.go | 2 +- main.go | 25 ++++++++-- output/output.go | 35 +++++++------- rules/yara.rules | 11 +++++ server/grpc.go | 116 ++++++++++++++++------------------------------- 11 files changed, 141 insertions(+), 174 deletions(-) create mode 100644 rules/yara.rules diff --git a/Dockerfile b/Dockerfile index db3d8a9..701ada7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,49 @@ MAINTAINER DeepFence RUN apk update \ && apk add --upgrade gcc musl-dev pkgconfig g++ make git -COPY --from=vectorscan /vectorscan.tar.bz2 / -RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2 +RUN apk add --no-cache \ + git \ + make \ + build-base \ + pkgconfig \ + libpcap-dev \ + libcap-dev \ + openssl-dev \ + file \ + jansson-dev \ + jansson-static \ + bison \ + tini \ + su-exec + +RUN apk add --no-cache -t .build-deps py-setuptools \ + openssl-libs-static \ + jansson-dev \ + build-base \ + libc-dev \ + file-dev \ + automake \ + autoconf \ + libtool \ + libcrypto3 \ + flex \ + git \ + libmagic-static \ + linux-headers + +RUN cd /root && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \ + && tar -zxf v4.3.2.tar.gz \ + && cd yara-4.3.2 \ + && ./bootstrap.sh \ + && ./configure --prefix=/usr/local/yara --disable-dotnet --enable-magic --enable-cuckoo --disable-shared --enable-static\ + && make \ + && make install \ + && cd /usr/local/ \ + && tar -czf yara.tar.gz yara WORKDIR /home/deepfence/src/SecretScanner COPY . . -RUN make clean -RUN make +RUN make clean && make all FROM alpine:3.18 MAINTAINER DeepFence @@ -30,7 +66,7 @@ RUN apk add --no-cache --upgrade tar libstdc++ libgcc docker skopeo bash podman RUN < ./agent-plugins-grpc -replace github.com/deepfence/YaraHunter => ../YaraHunter - require ( - github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000 + github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 diff --git a/go.sum b/go.sum index 6270cf7..0903485 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 h1:VuKkE8OKmM+hII/fViwYuLL1DTq5ncczMKWuOKg+EZc= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1/go.mod h1:SSZ34MXU0qR30p6/rJ/PpY7OgwljWF3go6K/ZfYqHZ0= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA= diff --git a/jobs/common.go b/jobs/common.go index a29b216..feb8441 100644 --- a/jobs/common.go +++ b/jobs/common.go @@ -6,8 +6,8 @@ import ( ) var ( - scanStatusFilename = getDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" - scanFilename = getDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" + scanStatusFilename = GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" + scanFilename = GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" SecretScanDir = "/" ) diff --git a/jobs/scan.go b/jobs/scan.go index 61606b4..4bde5e0 100644 --- a/jobs/scan.go +++ b/jobs/scan.go @@ -2,77 +2,15 @@ package jobs import ( "encoding/json" - "fmt" "strings" "sync" - "time" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) var ScanMap sync.Map -func DispatchScan(r *pb.FindRequest) { - go func() { - startScanJob() - defer stopScanJob() - - var err error - res, scanCtx := tasks.StartStatusReporter( - r.ScanId, - func(ss tasks.ScanStatus) error { - return writeSecretScanStatus(ss.ScanStatus, ss.ScanId, ss.ScanMessage) - }, - tasks.StatusValues{ - IN_PROGRESS: "IN_PROGRESS", - CANCELLED: "CANCELLED", - FAILED: "ERROR", - SUCCESS: "COMPLETE", - }, - time.Minute*20, - ) - - ScanMap.Store(r.ScanId, scanCtx) - - defer func() { - ScanMap.Delete(r.ScanId) - res <- err - close(res) - }() - - var ( - scanType scan.ScanType - nodeID string - ) - - if r.GetPath() != "" { - scanType = scan.DirScan - nodeID = r.GetPath() - } else if r.GetImage() != nil && r.GetImage().Name != "" { - scanType = scan.ImageScan - nodeID = r.GetImage().Name - } else if r.GetContainer() != nil && r.GetContainer().Id != "" { - scanType = scan.ContainerScan - nodeID = r.GetContainer().Id - } else { - err = fmt.Errorf("Invalid request") - return - } - - filters := cfg.Config2Filter(core.GetSession().ExtractorConfig) - err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) { - writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId) - }) - }() -} - type SecretScanDoc struct { pb.SecretInfo ScanID string `json:"scan_id,omitempty"` @@ -100,7 +38,7 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) { } } -func writeSingleScanData(secret *pb.SecretInfo, scan_id string) { +func WriteSingleScanData(secret *pb.SecretInfo, scan_id string) { if SecretScanDir == HostMountDir { secret.GetMatch().FullFilename = strings.Replace(secret.GetMatch().GetFullFilename(), SecretScanDir, "", 1) } diff --git a/jobs/status.go b/jobs/status.go index 856f200..81d2d5a 100644 --- a/jobs/status.go +++ b/jobs/status.go @@ -41,7 +41,7 @@ func writeScanDataToFile(secretScanMsg string, filename string) error { return nil } -func getDfInstallDir() string { +func GetDfInstallDir() string { installDir, exists := os.LookupEnv("DF_INSTALL_DIR") if exists { return installDir diff --git a/main.go b/main.go index c82101b..e33c628 100644 --- a/main.go +++ b/main.go @@ -37,14 +37,20 @@ import ( "time" "github.com/deepfence/SecretScanner/core" + "github.com/deepfence/SecretScanner/jobs" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/SecretScanner/signature" + "github.com/deepfence/SecretScanner/server" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + out "github.com/deepfence/YaraHunter/pkg/output" "github.com/deepfence/YaraHunter/pkg/runner" + yaraserver "github.com/deepfence/YaraHunter/pkg/server" + + pb "github.com/deepfence/agent-plugins-grpc/srcgo" ) const ( @@ -171,8 +177,6 @@ func main() { return "", " " + path.Base(f.File) + ":" + strconv.Itoa(f.Line) }, }) - // Process and store the read signatures - signature.ProcessSignatures(session.Config.Signatures) flag.Parse() @@ -182,6 +186,9 @@ func main() { ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) + out.ScanStatusFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan-log/secret_scan_log.log" + out.ScanFilename = jobs.GetDfInstallDir() + "/var/log/fenced/secret-scan/secret_scan.log" + runnerOpts := runner.RunnerOptions{ SocketPath: *socketPath, RulesPath: *core.GetSession().Options.RulesPath, @@ -206,5 +213,15 @@ func main() { go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts) } - runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig) + runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig, + + func(base *yaraserver.GRPCScannerServer) server.SecretGRPCServer { + return server.SecretGRPCServer{ + GRPCScannerServer: base, + UnimplementedSecretScannerServer: pb.UnimplementedSecretScannerServer{}, + } + }, + func(s grpc.ServiceRegistrar, impl any) { + pb.RegisterSecretScannerServer(s, impl.(pb.SecretScannerServer)) + }) } diff --git a/output/output.go b/output/output.go index 8db7250..f9d560e 100644 --- a/output/output.go +++ b/output/output.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/deepfence/YaraHunter/pkg/output" pb "github.com/deepfence/agent-plugins-grpc/srcgo" "github.com/fatih/color" tw "github.com/olekukonko/tablewriter" @@ -180,33 +181,31 @@ func removeFirstLastChar(input string) string { return input[1 : len(input)-1] } -func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo { - res := make([]*pb.SecretInfo, 0) - for _, v := range out { - res = append(res, SecretToSecretInfo(v)) +func SecretToSecretInfo(out output.IOCFound) *pb.SecretInfo { + matchedContent := "" + if len(out.Meta) != 0 { + matchedContent = out.Meta[0] + } + signature := "" + if len(out.StringsToMatch) != 0 { + signature = out.StringsToMatch[0] + } + severity := "low" + if out.Severity != "" { + severity = out.Severity } - return res -} - -func SecretToSecretInfo(out SecretFound) *pb.SecretInfo { return &pb.SecretInfo{ ImageLayerId: out.LayerID, Rule: &pb.MatchRule{ - Id: int32(out.RuleID), Name: out.RuleName, - Part: out.PartToMatch, - StringToMatch: out.Match, - SignatureToMatch: out.Regex, + SignatureToMatch: signature, }, Match: &pb.Match{ - StartingIndex: int64(out.PrintBufferStartIndex), - RelativeStartingIndex: int64(out.MatchFromByte), - RelativeEndingIndex: int64(out.MatchToByte), - FullFilename: out.CompleteFilename, - MatchedContent: jsonMarshal(out.MatchedContents), + FullFilename: out.CompleteFilename, + MatchedContent: matchedContent, }, Severity: &pb.Severity{ - Level: out.Severity, + Level: severity, Score: float32(out.SeverityScore), }, } diff --git a/rules/yara.rules b/rules/yara.rules new file mode 100644 index 0000000..088487a --- /dev/null +++ b/rules/yara.rules @@ -0,0 +1,11 @@ +rule github_personal_access_token { + meta: + description = "Rule to match GitHub Personal Access Tokens (classic), Fine-grained & Github Actions Token" + author = "deepfence.io" + + strings: + $github_pat = /^gh[ps]_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/ + + condition: + $github_pat +} diff --git a/server/grpc.go b/server/grpc.go index 0cdf8bd..7f6b081 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -2,95 +2,59 @@ package server import ( "context" - "fmt" - "net" - "sync" "github.com/deepfence/SecretScanner/jobs" + "github.com/deepfence/SecretScanner/output" + out "github.com/deepfence/YaraHunter/pkg/output" + "github.com/deepfence/YaraHunter/pkg/server" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - + "github.com/sirupsen/logrus" //nolint:typecheck - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - log "github.com/sirupsen/logrus" - "google.golang.org/grpc" ) -type gRPCServer struct { - socket_path string - plugin_name string +type SecretGRPCServer struct { + *server.GRPCScannerServer pb.UnimplementedSecretScannerServer - pb.UnimplementedAgentPluginServer - pb.UnimplementedScannersServer -} - -func (s *gRPCServer) ReportJobsStatus(context.Context, *pb.Empty) (*pb.JobReports, error) { - return &pb.JobReports{ - RunningJobs: jobs.GetRunningJobCount(), - }, nil -} - -func (s *gRPCServer) StopScan(c context.Context, req *pb.StopScanRequest) (*pb.StopScanResult, error) { - log.Errorf("Received StopScanRequest: %v", *req) - scanID := req.ScanId - result := &pb.StopScanResult{ - Success: true, - Description: "", - } - - obj, found := jobs.ScanMap.Load(scanID) - if !found { - log.Errorf("SecretScanner::Failed to Stop scan, may have already completed successfully or errored out, scan_id: %s", scanID) - result.Success = false - result.Description = "SecretScanner::Failed to Stop scan" - return result, nil - } else { - log.Errorf("SecretScanner::Stop request submitted") - result.Description = "SecretScanner::Stop request submitted" - } - - scanCtx := obj.(*tasks.ScanContext) - scanCtx.StopTriggered.Store(true) - scanCtx.Cancel() - return result, nil -} - -func (s *gRPCServer) GetName(context.Context, *pb.Empty) (*pb.Name, error) { - return &pb.Name{Str: s.plugin_name}, nil } -func (s *gRPCServer) GetUID(context.Context, *pb.Empty) (*pb.Uid, error) { - return &pb.Uid{Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path)}, nil -} - -func (s *gRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { - jobs.DispatchScan(r) - return &pb.FindResult{}, nil -} - -func RunServer(ctx context.Context, socket_path string, plugin_name string) error { - - lis, err := net.Listen("unix", fmt.Sprintf("%s", socket_path)) +func (s *SecretGRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { + yaraScanner, err := s.YaraRules.NewScanner() if err != nil { - return err + return &pb.FindResult{}, err } - s := grpc.NewServer() - jobs.ScanMap = sync.Map{} - - impl := &gRPCServer{socket_path: socket_path, plugin_name: plugin_name} - pb.RegisterAgentPluginServer(s, impl) - pb.RegisterSecretScannerServer(s, impl) - pb.RegisterScannersServer(s, impl) - log.Infof("main: server listening at %v", lis.Addr()) go func() { - if err := s.Serve(lis); err != nil { - log.Errorf("server: %v", err) + logrus.Infof("request to scan %+v", r) + + namespace := "" + container := "" + image := "" + path := "" + switch { + case r.GetContainer() != nil: + namespace = r.GetContainer().GetNamespace() + container = r.GetContainer().GetId() + case r.GetImage() != nil: + image = r.GetImage().GetName() + default: + path = r.GetPath() } - }() - - <-ctx.Done() - s.GracefulStop() - log.Infof("main: exiting gracefully") - return nil + server.DoScan( + r.ScanId, + s.HostMountPath, + s.ExtractorConfig, + s.InactiveThreshold, + &s.ScanMap, + namespace, + path, + image, + container, + yaraScanner, + func(res out.IOCFound, scanID string) { + jobs.WriteSingleScanData(output.SecretToSecretInfo(res), scanID) + }, + ) + }() + return &pb.FindResult{}, nil }