Skip to content

Commit

Permalink
feat: add assertion compiler (kyverno#503)
Browse files Browse the repository at this point in the history
Signed-off-by: Charles-Edouard Brétéché <[email protected]>
  • Loading branch information
eddycharly authored Sep 20, 2024
1 parent 7faca6c commit dc9a4be
Show file tree
Hide file tree
Showing 7 changed files with 90 additions and 19 deletions.
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ go 1.22.2
require (
github.com/aquilax/truncate v1.0.0
github.com/blang/semver/v4 v4.0.0
github.com/cespare/xxhash v1.1.0
github.com/elastic/go-freelru v0.13.0
github.com/gin-contrib/cors v1.7.2
github.com/gin-gonic/gin v1.10.0
github.com/google/cel-go v0.20.1
Expand All @@ -23,6 +25,7 @@ require (
gotest.tools v2.2.0+incompatible
k8s.io/apimachinery v0.31.1
k8s.io/client-go v0.31.1
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3
sigs.k8s.io/kubectl-validate v0.0.5-0.20240827210056-ce13d95db263
sigs.k8s.io/yaml v1.4.0
)
Expand Down Expand Up @@ -135,7 +138,6 @@ require (
k8s.io/component-base v0.31.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ github.com/IGLOU-EU/go-wildcard v1.0.3 h1:r8T46+8/9V1STciXJomTWRpPEv4nGJATDbJkdU
github.com/IGLOU-EU/go-wildcard v1.0.3/go.mod h1:/qeV4QLmydCbwH0UMQJmXDryrFKJknWi/jjO8IiuQfY=
github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/aquilax/truncate v1.0.0 h1:UgIGS8U/aZ4JyOJ2h3xcF5cSQ06+gGBnjxH2RUHJe0U=
Expand All @@ -24,6 +26,8 @@ github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
Expand All @@ -47,6 +51,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE=
github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
github.com/elastic/go-freelru v0.13.0 h1:TKKY6yCfNNNky7Pj9xZAOEpBcdNgZJfihEftOb55omg=
github.com/elastic/go-freelru v0.13.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I=
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down Expand Up @@ -237,6 +243,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
Expand Down
27 changes: 21 additions & 6 deletions pkg/apis/policy/v1alpha1/assertion_tree.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package v1alpha1

import (
"crypto/md5" //nolint:gosec
"encoding/hex"

"github.com/kyverno/kyverno-json/pkg/core/assertion"
"github.com/kyverno/kyverno-json/pkg/core/templating"
"k8s.io/apimachinery/pkg/util/json"
)

Expand All @@ -12,19 +14,30 @@ import (
// AssertionTree represents an assertion tree.
type AssertionTree struct {
_tree any
_hash string
}

func hash(in any) string {
if in == nil {
return ""
}
bytes, err := json.Marshal(in)
if err != nil {
return ""
}
hash := md5.Sum(bytes) //nolint:gosec
return hex.EncodeToString(hash[:])
}

func NewAssertionTree(value any) AssertionTree {
return AssertionTree{
_tree: value,
_hash: hash(value),
}
}

func (t *AssertionTree) Assertion(compiler templating.Compiler) (assertion.Assertion, error) {
if t._tree == nil {
return nil, nil
}
return assertion.Parse(t._tree, compiler)
func (t *AssertionTree) Compile(compiler func(string, any) (assertion.Assertion, error)) (assertion.Assertion, error) {
return compiler(t._hash, t._tree)
}

func (a *AssertionTree) MarshalJSON() ([]byte, error) {
Expand All @@ -38,9 +51,11 @@ func (a *AssertionTree) UnmarshalJSON(data []byte) error {
return err
}
a._tree = v
a._hash = hash(a._tree)
return nil
}

func (in *AssertionTree) DeepCopyInto(out *AssertionTree) {
out._tree = deepCopy(in._tree)
out._hash = in._hash
}
8 changes: 5 additions & 3 deletions pkg/json-engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@ func New() engine.Engine[Request, Response] {
resource any
bindings jpbinding.Bindings
}
compiler := templating.NewCompiler(templating.CompilerOptions{})
compiler := matching.Compiler{
Compiler: templating.NewCompiler(templating.CompilerOptions{}),
}
ruleEngine := builder.
Function(func(ctx context.Context, r ruleRequest) []RuleResponse {
bindings := r.bindings.Register("$rule", jpbinding.NewBinding(r.rule))
Expand All @@ -78,7 +80,7 @@ func New() engine.Engine[Request, Response] {
}
identifier := ""
if r.rule.Identifier != "" {
result, err := templating.ExecuteJP(r.rule.Identifier, r.resource, bindings, compiler)
result, err := templating.ExecuteJP(r.rule.Identifier, r.resource, bindings, compiler.Compiler)
if err != nil {
identifier = fmt.Sprintf("(error: %s)", err)
} else {
Expand Down Expand Up @@ -117,7 +119,7 @@ func New() engine.Engine[Request, Response] {
}
var feedback map[string]Feedback
for _, f := range r.rule.Feedback {
result, err := templating.ExecuteJP(f.Value, r.resource, bindings, compiler)
result, err := templating.ExecuteJP(f.Value, r.resource, bindings, compiler.Compiler)
if feedback == nil {
feedback = map[string]Feedback{}
}
Expand Down
44 changes: 44 additions & 0 deletions pkg/matching/compiler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package matching

import (
"sync"

"github.com/cespare/xxhash"
"github.com/elastic/go-freelru"
"github.com/kyverno/kyverno-json/pkg/core/assertion"
"github.com/kyverno/kyverno-json/pkg/core/templating"
)

type Compiler struct {
templating.Compiler
*freelru.SyncedLRU[string, func() (assertion.Assertion, error)]
}

func hashStringXXHASH(s string) uint32 {
sum := xxhash.Sum64String(s)
return uint32(sum) //nolint:gosec
}

func NewCompiler(compiler templating.Compiler, cacheSize uint32) Compiler {
out := Compiler{
Compiler: compiler,
}
if cache, err := freelru.NewSynced[string, func() (assertion.Assertion, error)](cacheSize, hashStringXXHASH); err == nil {
out.SyncedLRU = cache
}
return out
}

func (c Compiler) CompileAssertion(hash string, value any) (assertion.Assertion, error) {
if c.SyncedLRU == nil {
return assertion.Parse(value, c.Compiler)
}
entry, _ := c.SyncedLRU.Get(hash)
if entry == nil {
entry = sync.OnceValues(func() (assertion.Assertion, error) {
return assertion.Parse(value, c.Compiler)
})
c.SyncedLRU.Add(hash, entry)
}
return entry()
}
17 changes: 8 additions & 9 deletions pkg/matching/match.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (

"github.com/jmespath-community/go-jmespath/pkg/binding"
"github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1"
"github.com/kyverno/kyverno-json/pkg/core/templating"
"k8s.io/apimachinery/pkg/util/validation/field"
)

Expand Down Expand Up @@ -37,7 +36,7 @@ func (r Results) Error() string {
return strings.Join(lines, "\n")
}

func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings binding.Bindings, compiler templating.Compiler) ([]Result, error) {
func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings binding.Bindings, compiler Compiler) ([]Result, error) {
if len(match.Any) == 0 && len(match.All) == 0 {
return nil, field.Invalid(path, match, "an empty assert is not valid")
} else {
Expand All @@ -46,7 +45,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b
path := path.Child("any")
for i, assertion := range match.Any {
path := path.Index(i).Child("check")
parsed, err := assertion.Check.Assertion(compiler)
parsed, err := assertion.Check.Compile(compiler.CompileAssertion)
if err != nil {
return fails, err
}
Expand Down Expand Up @@ -75,7 +74,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b
path := path.Child("all")
for i, assertion := range match.All {
path := path.Index(i).Child("check")
parsed, err := assertion.Check.Assertion(compiler)
parsed, err := assertion.Check.Compile(compiler.CompileAssertion)
if err != nil {
return fails, err
}
Expand All @@ -99,7 +98,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b
}
}

func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) {
func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) {
if match == nil || (len(match.Any) == 0 && len(match.All) == 0) {
return nil, field.Invalid(path, match, "an empty match is not valid")
} else {
Expand All @@ -122,11 +121,11 @@ func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding
}
}

func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) {
func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) {
var errs field.ErrorList
for i, assertion := range assertions {
path := path.Index(i)
assertion, err := assertion.Assertion(compiler)
assertion, err := assertion.Compile(compiler.CompileAssertion)
if err != nil {
return errs, err
}
Expand All @@ -142,11 +141,11 @@ func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any,
return errs, nil
}

func MatchAll(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) {
func MatchAll(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) {
var errs field.ErrorList
for i, assertion := range assertions {
path := path.Index(i)
assertion, err := assertion.Assertion(compiler)
assertion, err := assertion.Compile(compiler.CompileAssertion)
if err != nil {
return errs, err
}
Expand Down
1 change: 1 addition & 0 deletions website/apis/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ hiddenMemberFields:
- _tree
- _assertion
- _message
- _hash

externalPackages:
- match: ^k8s\.io/apimachinery/pkg/apis/meta/v1\.Duration$
Expand Down

0 comments on commit dc9a4be

Please sign in to comment.