From a5e1812982c147700221ba83ec1e93db1c87f3ff Mon Sep 17 00:00:00 2001 From: Zhvanko Aleksander Date: Mon, 29 Apr 2024 09:36:33 +0200 Subject: [PATCH 1/4] feat(transformers): added new transformer which transform ip address Added a new transformer that generates a random IP address based on the provided CIDR --- internal/db/postgres/transformers/ip.go | 138 ++++++++++++++++++++++++ internal/generators/transformers/ip.go | 91 ++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 internal/db/postgres/transformers/ip.go create mode 100644 internal/generators/transformers/ip.go diff --git a/internal/db/postgres/transformers/ip.go b/internal/db/postgres/transformers/ip.go new file mode 100644 index 00000000..a1150d92 --- /dev/null +++ b/internal/db/postgres/transformers/ip.go @@ -0,0 +1,138 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformers + +import ( + "context" + "fmt" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" + "github.com/greenmaskio/greenmask/internal/generators/transformers" + "github.com/greenmaskio/greenmask/pkg/toolkit" +) + +var IpTransformerDefinition = utils.NewTransformerDefinition( + utils.NewTransformerProperties("IpTransformer", "Generate ip in the provided subnet"), + + NewIpTransformer, + + toolkit.MustNewParameterDefinition( + "column", + "column name", + ).SetIsColumn(toolkit.NewColumnProperties(). + SetAffected(true). + SetAllowedColumnTypes("text", "inet"), + ).SetRequired(true), + + toolkit.MustNewParameterDefinition( + "subnet", + "cidr subnet", + ).SetRequired(true), + + keepNullParameterDefinition, + + engineParameterDefinition, +) + +type IpTransformer struct { + columnName string + keepNull bool + affectedColumns map[int]string + columnIdx int + t *transformers.IpAddress +} + +func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { + var columnName, engine, subnet string + var keepNull bool + p := parameters["column"] + if err := p.Scan(&columnName); err != nil { + return nil, nil, fmt.Errorf(`unable to scan "column" param: %w`, err) + } + + idx, _, ok := driver.GetColumnByName(columnName) + if !ok { + return nil, nil, fmt.Errorf("column with name %s is not found", columnName) + } + affectedColumns := make(map[int]string) + affectedColumns[idx] = columnName + + p = parameters["keep_null"] + if err := p.Scan(&keepNull); err != nil { + return nil, nil, fmt.Errorf(`unable to scan "keep_null" param: %w`, err) + } + + p = parameters["engine"] + if err := p.Scan(&engine); err != nil { + return nil, nil, fmt.Errorf(`unable to scan "engine" param: %w`, err) + } + + p = parameters["subnet"] + if err := p.Scan(&subnet); err != nil { + return nil, nil, fmt.Errorf(`unable to scan "subnet" param: %w`, err) + } + + t := transformers.NewIpAddress(subnet) + g, err := getGenerateEngine(ctx, engine, t.GetRequiredGeneratorByteLength()) + if err != nil { + return nil, nil, fmt.Errorf("unable to get generator: %w", err) + } + if err = t.SetGenerator(g); err != nil { + return nil, nil, fmt.Errorf("unable to set generator: %w", err) + } + + return &IpTransformer{ + columnName: columnName, + keepNull: keepNull, + affectedColumns: affectedColumns, + columnIdx: idx, + t: t, + }, nil, nil +} + +func (rbt *IpTransformer) GetAffectedColumns() map[int]string { + return rbt.affectedColumns +} + +func (rbt *IpTransformer) Init(ctx context.Context) error { + return nil +} + +func (rbt *IpTransformer) Done(ctx context.Context) error { + return nil +} + +func (rbt *IpTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { + val, err := r.GetRawColumnValueByIdx(rbt.columnIdx) + if err != nil { + return nil, fmt.Errorf("unable to scan value: %w", err) + } + if val.IsNull && rbt.keepNull { + return r, nil + } + + ipVal, err := rbt.t.Generate() + if err != nil { + return nil, fmt.Errorf("unable to transform value: %w", err) + } + + if err = r.SetColumnValueByIdx(rbt.columnIdx, ipVal); err != nil { + return nil, fmt.Errorf("unable to set new value: %w", err) + } + return r, nil +} + +func init() { + utils.DefaultTransformerRegistry.MustRegister(IpTransformerDefinition) +} diff --git a/internal/generators/transformers/ip.go b/internal/generators/transformers/ip.go new file mode 100644 index 00000000..f0eefcc1 --- /dev/null +++ b/internal/generators/transformers/ip.go @@ -0,0 +1,91 @@ +// Copyright 2023 Greenmask +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformers + +import ( + "fmt" + "github.com/greenmaskio/greenmask/internal/generators" + "math/rand" + "net" + "time" +) + +type IpAddress struct { + subnet string + generator generators.Generator + byteLength int +} + +func NewIpAddress(subnet string) *IpAddress { + return &IpAddress{ + byteLength: 1, + subnet: subnet, + } +} + +func (b *IpAddress) GetRequiredGeneratorByteLength() int { + return b.byteLength +} + +func (b *IpAddress) Generate() (string, error) { + randomIP, err := randomIPInSubnet(b.subnet) + if err != nil { + fmt.Println("Ошибка:", err) + return "", nil + } + return randomIP.String(), err +} + +func randomIPInSubnet(cidr string) (net.IP, error) { + _, subnet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, err + } + + netIP := subnet.IP + mask := subnet.Mask + hostIP := make([]byte, len(netIP)) + + src := rand.NewSource(time.Now().UnixNano()) + rng := rand.New(src) + + for i := 0; i < len(hostIP); i++ { + val := byte(rng.Intn(256) & ^int(mask[i])) + + if val == 1 { + val += 1 + } + + if val == 255 { + val -= 1 + } + + hostIP[i] = byte(rng.Intn(256) & ^int(mask[i])) + } + + for i := range netIP { + netIP[i] |= hostIP[i] + } + + return netIP, nil +} + +func (b *IpAddress) SetGenerator(g generators.Generator) error { + if g.Size() < b.byteLength { + return fmt.Errorf("requested byte length (%d) higher than generator can produce (%d)", b.byteLength, g.Size()) + } + b.generator = g + return nil +} From 2b380033cb6544e282a7a08b28a6ba51d224f107 Mon Sep 17 00:00:00 2001 From: Zhvanko Aleksander Date: Fri, 10 May 2024 17:09:26 +0200 Subject: [PATCH 2/4] feat: refactored logic and added tests --- internal/db/postgres/transformers/ip.go | 66 +++++++++---- internal/generators/transformers/ip.go | 51 +++++----- internal/generators/transformers/ip_test.go | 102 ++++++++++++++++++++ 3 files changed, 174 insertions(+), 45 deletions(-) create mode 100644 internal/generators/transformers/ip_test.go diff --git a/internal/db/postgres/transformers/ip.go b/internal/db/postgres/transformers/ip.go index a1150d92..dc4f269e 100644 --- a/internal/db/postgres/transformers/ip.go +++ b/internal/db/postgres/transformers/ip.go @@ -17,13 +17,15 @@ package transformers import ( "context" "fmt" + "net" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/generators/transformers" "github.com/greenmaskio/greenmask/pkg/toolkit" ) -var IpTransformerDefinition = utils.NewTransformerDefinition( - utils.NewTransformerProperties("IpTransformer", "Generate ip in the provided subnet"), +var RandomIpDefinition = utils.NewTransformerDefinition( + utils.NewTransformerProperties("RandomIp", "Generate ip in the provided subnet"), NewIpTransformer, @@ -32,30 +34,42 @@ var IpTransformerDefinition = utils.NewTransformerDefinition( "column name", ).SetIsColumn(toolkit.NewColumnProperties(). SetAffected(true). - SetAllowedColumnTypes("text", "inet"), + SetAllowedColumnTypes("text", "varchar", "inet"), ).SetRequired(true), toolkit.MustNewParameterDefinition( "subnet", "cidr subnet", - ).SetRequired(true), + ).SetRequired(true). + SetCastDbType("cidr"). + SetDynamicMode( + toolkit.NewDynamicModeProperties(). + SetCompatibleTypes("cidr"), + ), keepNullParameterDefinition, engineParameterDefinition, ) -type IpTransformer struct { +type RandomIp struct { columnName string keepNull bool affectedColumns map[int]string columnIdx int + dynamicMode bool t *transformers.IpAddress + subnetParam toolkit.Parameterizer } func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { - var columnName, engine, subnet string + + subnetParam := parameters["subnet"] + + var columnName, engine string + var subnet *net.IPNet var keepNull bool + var dynamicMode bool p := parameters["column"] if err := p.Scan(&columnName); err != nil { return nil, nil, fmt.Errorf(`unable to scan "column" param: %w`, err) @@ -78,12 +92,19 @@ func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters ma return nil, nil, fmt.Errorf(`unable to scan "engine" param: %w`, err) } - p = parameters["subnet"] - if err := p.Scan(&subnet); err != nil { - return nil, nil, fmt.Errorf(`unable to scan "subnet" param: %w`, err) + if subnetParam.IsDynamic() { + dynamicMode = true + } else { + subnet = &net.IPNet{} + if err := subnetParam.Scan(subnet); err != nil { + return nil, nil, fmt.Errorf(`unable to scan "subnet" param: %w`, err) + } } - t := transformers.NewIpAddress(subnet) + t, err := transformers.NewIpAddress(subnet) + if err != nil { + return nil, nil, fmt.Errorf("unable to create ip transformer: %w", err) + } g, err := getGenerateEngine(ctx, engine, t.GetRequiredGeneratorByteLength()) if err != nil { return nil, nil, fmt.Errorf("unable to get generator: %w", err) @@ -92,28 +113,31 @@ func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters ma return nil, nil, fmt.Errorf("unable to set generator: %w", err) } - return &IpTransformer{ + return &RandomIp{ columnName: columnName, keepNull: keepNull, affectedColumns: affectedColumns, columnIdx: idx, t: t, + subnetParam: subnetParam, + dynamicMode: dynamicMode, }, nil, nil } -func (rbt *IpTransformer) GetAffectedColumns() map[int]string { +func (rbt *RandomIp) GetAffectedColumns() map[int]string { return rbt.affectedColumns } -func (rbt *IpTransformer) Init(ctx context.Context) error { +func (rbt *RandomIp) Init(ctx context.Context) error { return nil } -func (rbt *IpTransformer) Done(ctx context.Context) error { +func (rbt *RandomIp) Done(ctx context.Context) error { return nil } -func (rbt *IpTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { +func (rbt *RandomIp) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { + val, err := r.GetRawColumnValueByIdx(rbt.columnIdx) if err != nil { return nil, fmt.Errorf("unable to scan value: %w", err) @@ -122,7 +146,15 @@ func (rbt *IpTransformer) Transform(ctx context.Context, r *toolkit.Record) (*to return r, nil } - ipVal, err := rbt.t.Generate() + var subnet *net.IPNet + if rbt.dynamicMode { + subnet = &net.IPNet{} + if err = rbt.subnetParam.Scan(subnet); err != nil { + return nil, fmt.Errorf(`unable to scan "subnet" param: %w`, err) + } + } + + ipVal, err := rbt.t.Generate(val.Data, subnet) if err != nil { return nil, fmt.Errorf("unable to transform value: %w", err) } @@ -134,5 +166,5 @@ func (rbt *IpTransformer) Transform(ctx context.Context, r *toolkit.Record) (*to } func init() { - utils.DefaultTransformerRegistry.MustRegister(IpTransformerDefinition) + utils.DefaultTransformerRegistry.MustRegister(RandomIpDefinition) } diff --git a/internal/generators/transformers/ip.go b/internal/generators/transformers/ip.go index f0eefcc1..5fe1417b 100644 --- a/internal/generators/transformers/ip.go +++ b/internal/generators/transformers/ip.go @@ -16,63 +16,58 @@ package transformers import ( "fmt" - "github.com/greenmaskio/greenmask/internal/generators" - "math/rand" "net" - "time" + + "github.com/greenmaskio/greenmask/internal/generators" ) type IpAddress struct { - subnet string + subnet *net.IPNet generator generators.Generator byteLength int } -func NewIpAddress(subnet string) *IpAddress { +func NewIpAddress(subnet *net.IPNet) (*IpAddress, error) { return &IpAddress{ - byteLength: 1, + byteLength: 16, subnet: subnet, - } + }, nil } func (b *IpAddress) GetRequiredGeneratorByteLength() int { return b.byteLength } -func (b *IpAddress) Generate() (string, error) { - randomIP, err := randomIPInSubnet(b.subnet) +func (b *IpAddress) Generate(original []byte, runtimeSubnet *net.IPNet) (net.IP, error) { + + subnet := b.subnet + if runtimeSubnet != nil { + subnet = runtimeSubnet + } + + randomBytes, err := b.generator.Generate(original) if err != nil { - fmt.Println("Ошибка:", err) - return "", nil + return nil, fmt.Errorf("error generating random bytes: %w", err) } - return randomIP.String(), err -} -func randomIPInSubnet(cidr string) (net.IP, error) { - _, subnet, err := net.ParseCIDR(cidr) + randomIP, err := randomIPInSubnet(subnet, randomBytes) if err != nil { - return nil, err + return nil, fmt.Errorf("error generating random IP: %w", err) } + return randomIP, err +} + +func randomIPInSubnet(subnet *net.IPNet, randomBytes []byte) (net.IP, error) { netIP := subnet.IP mask := subnet.Mask hostIP := make([]byte, len(netIP)) - src := rand.NewSource(time.Now().UnixNano()) - rng := rand.New(src) - for i := 0; i < len(hostIP); i++ { - val := byte(rng.Intn(256) & ^int(mask[i])) - - if val == 1 { - val += 1 - } - if val == 255 { - val -= 1 - } + val := randomBytes[i] & ^mask[i] - hostIP[i] = byte(rng.Intn(256) & ^int(mask[i])) + hostIP[i] = val } for i := range netIP { diff --git a/internal/generators/transformers/ip_test.go b/internal/generators/transformers/ip_test.go new file mode 100644 index 00000000..bdb01b08 --- /dev/null +++ b/internal/generators/transformers/ip_test.go @@ -0,0 +1,102 @@ +package transformers + +import ( + "net" + "testing" + "time" + + "github.com/greenmaskio/greenmask/internal/generators" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +func TestIpAddress_Generate(t *testing.T) { + type test struct { + name string + subnet string + dynamic bool + } + + tests := []test{ + { + name: "static v4", + subnet: "192.168.1.0/24", + dynamic: false, + }, + { + name: "dynamic v4", + subnet: "192.168.1.0/24", + dynamic: true, + }, + { + name: "static v6", + subnet: "2001:0db8:85a3::/64", + dynamic: false, + }, + { + name: "dynamic v6", + subnet: "2001:0db8:85a3::/64", + dynamic: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, subnet, err := net.ParseCIDR(tt.subnet) + require.NoError(t, err) + tr, err := NewIpAddress(subnet) + require.NoError(t, err) + g := generators.NewRandomBytes(time.Now().UnixNano(), tr.GetRequiredGeneratorByteLength()) + require.NoError(t, err) + err = tr.SetGenerator(g) + + var res net.IP + if tt.dynamic { + res, err = tr.Generate([]byte{}, subnet) + } else { + res, err = tr.Generate([]byte{}, nil) + } + require.NoError(t, err) + log.Debug(). + Str("IP", res.String()). + Str("subnet", subnet.String()). + Msg("results") + require.True(t, subnet.Contains(res)) + }) + } +} + +func TestIpAddress_Generate_check_address_is_not_subnet_and_broadcast(t *testing.T) { + + broadcast := net.ParseIP("192.168.1.3") + _, subnet, err := net.ParseCIDR("192.168.1.0/30") + require.NoError(t, err) + tr, err := NewIpAddress(subnet) + + for i := 0; i < 100000; i++ { + require.NoError(t, err) + g := generators.NewRandomBytes(time.Now().UnixNano(), tr.GetRequiredGeneratorByteLength()) + require.NoError(t, err) + err = tr.SetGenerator(g) + res, err := tr.Generate([]byte{}, nil) + require.NoError(t, err) + require.True(t, !res.Equal(broadcast) && !res.Equal(subnet.IP), "IP address is subnet or broadcast") + } +} + +func BenchmarkIpAddress_Generate(b *testing.B) { + broadcast := net.ParseIP("192.168.1.3") + _, subnet, err := net.ParseCIDR("192.168.1.0/30") + require.NoError(b, err) + tr, err := NewIpAddress(subnet) + g := generators.NewRandomBytes(time.Now().UnixNano(), tr.GetRequiredGeneratorByteLength()) + + for i := 0; i < b.N; i++ { + require.NoError(b, err) + require.NoError(b, err) + err = tr.SetGenerator(g) + res, err := tr.Generate([]byte{}, nil) + require.NoError(b, err) + require.True(b, !res.Equal(broadcast) && !res.Equal(subnet.IP), "IP address is subnet or broadcast") + } +} From 15df87a0a3a6514c3d2c9d1de0dfcdf70897bc64 Mon Sep 17 00:00:00 2001 From: Zhvanko Aleksander Date: Fri, 10 May 2024 18:03:24 +0200 Subject: [PATCH 3/4] fix: changed ip generator --- internal/generators/transformers/ip.go | 71 +++++++++++---------- internal/generators/transformers/ip_test.go | 1 + 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/internal/generators/transformers/ip.go b/internal/generators/transformers/ip.go index 5fe1417b..34e6eb13 100644 --- a/internal/generators/transformers/ip.go +++ b/internal/generators/transformers/ip.go @@ -16,6 +16,7 @@ package transformers import ( "fmt" + "math/big" "net" "github.com/greenmaskio/greenmask/internal/generators" @@ -34,53 +35,57 @@ func NewIpAddress(subnet *net.IPNet) (*IpAddress, error) { }, nil } -func (b *IpAddress) GetRequiredGeneratorByteLength() int { - return b.byteLength +func (ip *IpAddress) GetRequiredGeneratorByteLength() int { + return ip.byteLength } -func (b *IpAddress) Generate(original []byte, runtimeSubnet *net.IPNet) (net.IP, error) { - - subnet := b.subnet +func (ip *IpAddress) Generate(original []byte, runtimeSubnet *net.IPNet) (net.IP, error) { + subnet := ip.subnet if runtimeSubnet != nil { subnet = runtimeSubnet } + ones, bits := subnet.Mask.Size() + totalHosts := big.NewInt(1) + totalHosts.Lsh(totalHosts, uint(bits-ones)) - randomBytes, err := b.generator.Generate(original) - if err != nil { - return nil, fmt.Errorf("error generating random bytes: %w", err) + if totalHosts.Cmp(big.NewInt(2)) <= 0 { + return nil, fmt.Errorf("subnet too small") } - randomIP, err := randomIPInSubnet(subnet, randomBytes) + // Generate random host part within the range, avoiding special addresses + randomHostNum := big.NewInt(0) + + hostBytes, err := ip.generator.Generate(original) if err != nil { - return nil, fmt.Errorf("error generating random IP: %w", err) + return nil, fmt.Errorf("error generating random bytes: %w", err) } - return randomIP, err -} - -func randomIPInSubnet(subnet *net.IPNet, randomBytes []byte) (net.IP, error) { - - netIP := subnet.IP - mask := subnet.Mask - hostIP := make([]byte, len(netIP)) - - for i := 0; i < len(hostIP); i++ { - - val := randomBytes[i] & ^mask[i] - - hostIP[i] = val + if subnet.IP.To4() != nil { + hostBytes = hostBytes[:4] // Use only the first 4 bytes for IPv4 } - - for i := range netIP { - netIP[i] |= hostIP[i] + // IPv6, use all 16 bytes + randomHostNum.SetBytes(hostBytes) + randomHostNum.Mod(randomHostNum, new(big.Int).Sub(totalHosts, big.NewInt(2))) // [0, totalHosts-3] + randomHostNum.Add(randomHostNum, big.NewInt(1)) // [1, totalHosts-2] + + // Calculate the IP address + networkInt := big.NewInt(0) + networkInt.SetBytes(subnet.IP) + networkInt.Add(networkInt, randomHostNum) + + ipAddrBytes := networkInt.Bytes() + if len(ipAddrBytes) < 16 && subnet.IP.To4() == nil { + // Pad the address to 16 bytes if it's an IPv6 address + paddedIP := make([]byte, 16) + copy(paddedIP[16-len(ipAddrBytes):], ipAddrBytes) + return paddedIP, nil } - - return netIP, nil + return ipAddrBytes, nil } -func (b *IpAddress) SetGenerator(g generators.Generator) error { - if g.Size() < b.byteLength { - return fmt.Errorf("requested byte length (%d) higher than generator can produce (%d)", b.byteLength, g.Size()) +func (ip *IpAddress) SetGenerator(g generators.Generator) error { + if g.Size() < ip.byteLength { + return fmt.Errorf("requested byte length (%d) higher than generator can produce (%d)", ip.byteLength, g.Size()) } - b.generator = g + ip.generator = g return nil } diff --git a/internal/generators/transformers/ip_test.go b/internal/generators/transformers/ip_test.go index bdb01b08..052e17e6 100644 --- a/internal/generators/transformers/ip_test.go +++ b/internal/generators/transformers/ip_test.go @@ -89,6 +89,7 @@ func BenchmarkIpAddress_Generate(b *testing.B) { _, subnet, err := net.ParseCIDR("192.168.1.0/30") require.NoError(b, err) tr, err := NewIpAddress(subnet) + require.NoError(b, err) g := generators.NewRandomBytes(time.Now().UnixNano(), tr.GetRequiredGeneratorByteLength()) for i := 0; i < b.N; i++ { From c7f6358a8f1087b3213544a57ae7a29b9b724313 Mon Sep 17 00:00:00 2001 From: Zhvanko Aleksander Date: Sat, 11 May 2024 16:52:20 +0200 Subject: [PATCH 4/4] feat(tests): added new tests for ip transformer feat(random-ip): rename ip transformer feat(documentation): changed documentation for random ip transformer. Remove documentation for ipv6 --- .../standard_transformers/index.md | 3 +- .../standard_transformers/random_ip.md | 75 ++++++++++++++++++ .../standard_transformers/random_ipv4.md | 28 ------- .../standard_transformers/random_ipv6.md | 28 ------- .../transformers/{ip.go => random_ip.go} | 24 ++---- .../postgres/transformers/random_ip_test.go | 77 +++++++++++++++++++ .../transformers/{ip.go => random_ip.go} | 0 .../{ip_test.go => random_ip_test.go} | 5 +- mkdocs.yml | 3 +- pkg/toolkit/tesing_helpers.go | 8 ++ 10 files changed, 170 insertions(+), 81 deletions(-) create mode 100644 docs/built_in_transformers/standard_transformers/random_ip.md delete mode 100644 docs/built_in_transformers/standard_transformers/random_ipv4.md delete mode 100644 docs/built_in_transformers/standard_transformers/random_ipv6.md rename internal/db/postgres/transformers/{ip.go => random_ip.go} (88%) create mode 100644 internal/db/postgres/transformers/random_ip_test.go rename internal/generators/transformers/{ip.go => random_ip.go} (100%) rename internal/generators/transformers/{ip_test.go => random_ip_test.go} (96%) diff --git a/docs/built_in_transformers/standard_transformers/index.md b/docs/built_in_transformers/standard_transformers/index.md index 29882003..97c9abd3 100644 --- a/docs/built_in_transformers/standard_transformers/index.md +++ b/docs/built_in_transformers/standard_transformers/index.md @@ -31,8 +31,7 @@ Standard transformers are ready-to-use methods that require no customization and 1. [RandomMacAddress](random_mac_address.md) — generates a random MAC address. 1. [RandomDomainName](random_domain_name.md) — generates a random domain name. 1. [RandomURL](random_url.md) — generates a random URL. -1. [RandomIPv4](random_ipv4.md) — generates a random IPv4 address. -1. [RandomIPv6](random_ipv6.md) — generates a random IPv6 address. +1. [RandomIP](random_ip.md) — generates a random IPv4 or IPv6 addresses. 1. [RandomWord](random_word.md) — generates a random word. 1. [RandomSentence](random_sentence.md) — generates a random sentence. 1. [RandomParagraph](random_paragraph.md) — generates a random paragraph. diff --git a/docs/built_in_transformers/standard_transformers/random_ip.md b/docs/built_in_transformers/standard_transformers/random_ip.md new file mode 100644 index 00000000..1810ea8e --- /dev/null +++ b/docs/built_in_transformers/standard_transformers/random_ip.md @@ -0,0 +1,75 @@ +The `RandomIp` transformer is designed to populate specified database columns with random IP v4 or V6 addresses. +This utility is essential for applications requiring the simulation of network data, testing systems that utilize IP +addresses, or anonymizing real IP addresses in datasets. + +## Parameters + +| Name | Description | Default | Required | Supported DB types | +|-----------|----------------------------------------------------|----------|----------|---------------------| +| column | The name of the column to be affected | | Yes | text, varchar, inet | +| subnet | Subnet for generating random ip in V4 or V6 format | `false` | No | - | +| engine | The engine used for generating the values [random, hash]. Use hash for deterministic generation | `random` |No |- | + +## Dynamic parameters + +| Name | Supported types | +|------------|---------------------| +| subnet | cidr, text, varchar | + +## Description + +Utilizing a robust algorithm or library for generating IP addresses, the `RandomIp` transformer injects random IPv4 +or IPv6 addresses into the designated database column, depending on the provided subnet. The transformer automatically +detects whether to generate an IPv4 or IPv6 address based on the subnet version specified. + + +## Example: Generate a Random IPv4 Address for a 192.168.1.0/24 Subnet + +This example demonstrates how to configure the RandomIp transformer to inject a random IPv4 address into the +ip_address column for entries in the `192.168.1.0/24` subnet: + +```sql title="Create table ip_networks and insert data" +CREATE TABLE ip_networks +( + id SERIAL PRIMARY KEY, + ip_address INET, + network CIDR +); + +INSERT INTO ip_networks (ip_address, network) +VALUES ('192.168.1.10', '192.168.1.0/24'), + ('10.0.0.5', '10.0.0.0/16'), + ('172.16.254.3', '172.16.0.0/12'), + ('192.168.100.14', '192.168.100.0/24'), + ('2001:0db8:85a3:0000:0000:8a2e:0370:7334', '2001:0db8:85a3::/64'); -- An IPv6 address and network + +``` + +```yaml title="RandomPerson transformer example" +- schema: public + name: ip_networks + transformers: + - name: "RandomIp" + params: + subnet: "192.168.1.0/24" + column: "ip_address" + engine: "random" +``` + +## Example: Generate a Random IP Based on the Dynamic Subnet Parameter + +This configuration illustrates how to use the RandomIp transformer dynamically, where it reads the subnet information +from the network column of the database and generates a corresponding random IP address: + +```yaml title="RandomPerson transformer example with dynamic mode" +- schema: public + name: ip_networks + transformers: + - name: "RandomIp" + params: + column: "ip_address" + engine: "random" + dynamic_params: + subnet: + column: "network" +``` \ No newline at end of file diff --git a/docs/built_in_transformers/standard_transformers/random_ipv4.md b/docs/built_in_transformers/standard_transformers/random_ipv4.md deleted file mode 100644 index 321554c6..00000000 --- a/docs/built_in_transformers/standard_transformers/random_ipv4.md +++ /dev/null @@ -1,28 +0,0 @@ -The `RandomIPv4` transformer is designed to populate specified database columns with random IPv4 addresses. This utility is essential for applications requiring the simulation of network data, testing systems that utilize IP addresses, or anonymizing real IP addresses in datasets. - -## Parameters - -| Name | Description | Default | Required | Supported DB types | -|-----------|------------------------------------------------------|----------|----------|---------------------| -| column | The name of the column to be affected | | Yes | text, varchar, inet | -| keep_null | Indicates whether NULL values should be preserved | `false` | No | - | - -## Description - -Utilizing a robust algorithm or library for generating IPv4 address strings, the `RandomIPv4` transformer injects random IPv4 addresses into the designated database column. Each generated address follows the standard IPv4 format, consisting of four octets separated by dots (e. g., "192.168.1.1"), ensuring a wide range of plausible network addresses for various use cases. - -## Example: Populate random IPv4 addresses for the `network_logs` table - -This example shows how to configure the `RandomIPv4` transformer to populate the `source_ip` column in the `network_logs` table with random IPv4 addresses, simulating diverse network traffic sources for analysis or testing purposes. - -```yaml title="RandomIPv4 transformer example" -- schema: "public" - name: "network_logs" - transformers: - - name: "RandomIPv4" - params: - column: "source_ip" - keep_null: false -``` - -With this setup, the `source_ip` column will be updated with random IPv4 addresses for each entry, replacing any existing non-NULL values. If the `keep_null` parameter is set to `true`, it will ensure that existing NULL values in the column are preserved, accommodating scenarios where IP address data may be intentionally omitted. diff --git a/docs/built_in_transformers/standard_transformers/random_ipv6.md b/docs/built_in_transformers/standard_transformers/random_ipv6.md deleted file mode 100644 index 0226b98c..00000000 --- a/docs/built_in_transformers/standard_transformers/random_ipv6.md +++ /dev/null @@ -1,28 +0,0 @@ -The `RandomIPv6` transformer is engineered to populate specified database columns with random IPv6 addresses. This tool is particularly useful for simulating modern network environments, testing systems that operate with IPv6 addresses, or anonymizing datasets containing real IPv6 addresses. - -## Parameters - -| Name | Description | Default | Required | Supported DB types | -|-----------|------------------------------------------------------|---------|----------|---------------------| -| column | The name of the column to be affected | | Yes | text, varchar, inet | -| keep_null | Indicates whether NULL values should be preserved | `false` | No | - | - -## Description - -Employing advanced algorithms or libraries capable of generating IPv6 address strings, the `RandomIPv6` transformer introduces random IPv6 addresses into the specified database column. IPv6 addresses, represented as eight groups of four hexadecimal digits separated by colons (e. g., "2001:0db8:85a3:0000:0000:8a2e:0370:7334"), provide a vast range of possible addresses, reflecting the extensive addressing capacity of the IPv6 standard. - -## Example: Populate random IPv6 addresses for the `devices` table - -This example illustrates configuring the `RandomIPv6` transformer to populate the `device_ip` column in the `devices` table with random IPv6 addresses, enhancing the dataset with a broad spectrum of network addresses for development, testing, or data protection purposes. - -```yaml title="RandomIPv6 transformer example" -- schema: "public" - name: "devices" - transformers: - - name: "RandomIPv6" - params: - column: "device_ip" - keep_null: false -``` - -This configuration ensures that the `device_ip` column receives random IPv6 addresses for each entry, replacing any existing non-NULL values. Setting the `keep_null` parameter to `true` allows for the preservation of existing NULL values within the column, maintaining the integrity of records where IP address information is not applicable or available. diff --git a/internal/db/postgres/transformers/ip.go b/internal/db/postgres/transformers/random_ip.go similarity index 88% rename from internal/db/postgres/transformers/ip.go rename to internal/db/postgres/transformers/random_ip.go index dc4f269e..96f454bf 100644 --- a/internal/db/postgres/transformers/ip.go +++ b/internal/db/postgres/transformers/random_ip.go @@ -25,13 +25,13 @@ import ( ) var RandomIpDefinition = utils.NewTransformerDefinition( - utils.NewTransformerProperties("RandomIp", "Generate ip in the provided subnet"), + utils.NewTransformerProperties("RandomIp", "Generate V4 or V6 IP in the provided subnet"), NewIpTransformer, toolkit.MustNewParameterDefinition( "column", - "column name", + "Column name", ).SetIsColumn(toolkit.NewColumnProperties(). SetAffected(true). SetAllowedColumnTypes("text", "varchar", "inet"), @@ -39,22 +39,19 @@ var RandomIpDefinition = utils.NewTransformerDefinition( toolkit.MustNewParameterDefinition( "subnet", - "cidr subnet", + "Subnet for generating random ip in V4 or V6 format", ).SetRequired(true). SetCastDbType("cidr"). SetDynamicMode( toolkit.NewDynamicModeProperties(). - SetCompatibleTypes("cidr"), + SetCompatibleTypes("cidr", "text", "varchar"), ), - keepNullParameterDefinition, - engineParameterDefinition, ) type RandomIp struct { columnName string - keepNull bool affectedColumns map[int]string columnIdx int dynamicMode bool @@ -68,7 +65,6 @@ func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters ma var columnName, engine string var subnet *net.IPNet - var keepNull bool var dynamicMode bool p := parameters["column"] if err := p.Scan(&columnName); err != nil { @@ -82,11 +78,6 @@ func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters ma affectedColumns := make(map[int]string) affectedColumns[idx] = columnName - p = parameters["keep_null"] - if err := p.Scan(&keepNull); err != nil { - return nil, nil, fmt.Errorf(`unable to scan "keep_null" param: %w`, err) - } - p = parameters["engine"] if err := p.Scan(&engine); err != nil { return nil, nil, fmt.Errorf(`unable to scan "engine" param: %w`, err) @@ -115,7 +106,6 @@ func NewIpTransformer(ctx context.Context, driver *toolkit.Driver, parameters ma return &RandomIp{ columnName: columnName, - keepNull: keepNull, affectedColumns: affectedColumns, columnIdx: idx, t: t, @@ -142,9 +132,6 @@ func (rbt *RandomIp) Transform(ctx context.Context, r *toolkit.Record) (*toolkit if err != nil { return nil, fmt.Errorf("unable to scan value: %w", err) } - if val.IsNull && rbt.keepNull { - return r, nil - } var subnet *net.IPNet if rbt.dynamicMode { @@ -159,7 +146,8 @@ func (rbt *RandomIp) Transform(ctx context.Context, r *toolkit.Record) (*toolkit return nil, fmt.Errorf("unable to transform value: %w", err) } - if err = r.SetColumnValueByIdx(rbt.columnIdx, ipVal); err != nil { + newRawValue := toolkit.NewRawValue([]byte(ipVal.String()), false) + if err = r.SetRawColumnValueByIdx(rbt.columnIdx, newRawValue); err != nil { return nil, fmt.Errorf("unable to set new value: %w", err) } return r, nil diff --git a/internal/db/postgres/transformers/random_ip_test.go b/internal/db/postgres/transformers/random_ip_test.go new file mode 100644 index 00000000..69f617f6 --- /dev/null +++ b/internal/db/postgres/transformers/random_ip_test.go @@ -0,0 +1,77 @@ +package transformers + +import ( + "context" + "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" + "github.com/greenmaskio/greenmask/pkg/toolkit" + "github.com/stretchr/testify/require" + "testing" +) + +func TestRandomIpTransformer_Transform_random_dynamic(t *testing.T) { + + tests := []struct { + name string + columnName string + params map[string]toolkit.ParamsValue + dynamicParams map[string]*toolkit.DynamicParamValue + record map[string]*toolkit.RawValue + expected string + }{ + { + name: "IPv4 dynamic test", + columnName: "data", + record: map[string]*toolkit.RawValue{ + "data": toolkit.NewRawValue([]byte("192.168.1.10"), false), + "data2": toolkit.NewRawValue([]byte("192.168.1.0/30"), false), + }, + params: map[string]toolkit.ParamsValue{ + "engine": toolkit.ParamsValue("random"), + }, + dynamicParams: map[string]*toolkit.DynamicParamValue{ + "subnet": { + Column: "data2", + }, + }, + expected: "192.168.1.[1,2]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + driver, record := toolkit.GetDriverAndRecord(tt.record) + + tt.params["column"] = toolkit.ParamsValue(tt.columnName) + def, ok := utils.DefaultTransformerRegistry.Get("RandomIp") + require.True(t, ok) + + transformer, warnings, err := def.Instance( + context.Background(), + driver, + tt.params, + tt.dynamicParams, + ) + require.NoError(t, err) + require.Empty(t, warnings) + + err = transformer.Transformer.Init(context.Background()) + require.NoError(t, err) + + for _, dp := range transformer.DynamicParameters { + dp.SetRecord(record) + } + + r, err := transformer.Transformer.Transform( + context.Background(), + record, + ) + require.NoError(t, err) + + rawVal, err := r.GetRawColumnValueByName(tt.columnName) + require.NoError(t, err) + require.False(t, rawVal.IsNull) + require.Regexp(t, tt.expected, string(rawVal.Data)) + }) + } +} diff --git a/internal/generators/transformers/ip.go b/internal/generators/transformers/random_ip.go similarity index 100% rename from internal/generators/transformers/ip.go rename to internal/generators/transformers/random_ip.go diff --git a/internal/generators/transformers/ip_test.go b/internal/generators/transformers/random_ip_test.go similarity index 96% rename from internal/generators/transformers/ip_test.go rename to internal/generators/transformers/random_ip_test.go index 052e17e6..55d17ef5 100644 --- a/internal/generators/transformers/ip_test.go +++ b/internal/generators/transformers/random_ip_test.go @@ -91,11 +91,10 @@ func BenchmarkIpAddress_Generate(b *testing.B) { tr, err := NewIpAddress(subnet) require.NoError(b, err) g := generators.NewRandomBytes(time.Now().UnixNano(), tr.GetRequiredGeneratorByteLength()) + err = tr.SetGenerator(g) + require.NoError(b, err) for i := 0; i < b.N; i++ { - require.NoError(b, err) - require.NoError(b, err) - err = tr.SetGenerator(g) res, err := tr.Generate([]byte{}, nil) require.NoError(b, err) require.True(b, !res.Equal(broadcast) && !res.Equal(subnet.IP), "IP address is subnet or broadcast") diff --git a/mkdocs.yml b/mkdocs.yml index c6574dfc..7c70955c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,8 +85,7 @@ nav: - RandomDomainName: built_in_transformers/standard_transformers/random_domain_name.md - RandomPerson: built_in_transformers/standard_transformers/random_person.md - RandomURL: built_in_transformers/standard_transformers/random_url.md - - RandomIPv4: built_in_transformers/standard_transformers/random_ipv4.md - - RandomIPv6: built_in_transformers/standard_transformers/random_ipv6.md + - RandomIP: built_in_transformers/standard_transformers/random_ip.md - RandomWord: built_in_transformers/standard_transformers/random_word.md - RandomSentence: built_in_transformers/standard_transformers/random_sentence.md - RandomParagraph: built_in_transformers/standard_transformers/random_paragraph.md diff --git a/pkg/toolkit/tesing_helpers.go b/pkg/toolkit/tesing_helpers.go index 31c71e46..ea13c52b 100644 --- a/pkg/toolkit/tesing_helpers.go +++ b/pkg/toolkit/tesing_helpers.go @@ -190,6 +190,14 @@ var columnList = []*Column{ NotNull: false, Length: -1, }, + { + Name: "data2", + TypeName: "text", + TypeOid: pgtype.TextOID, + Num: 22, + NotNull: false, + Length: -1, + }, } // GetDriverAndRecord - return adhoc table for testing