Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(compress): allow passing in compressor options #509

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions mutate/compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,15 @@ type Compressor interface {
// indicate what compression type is used, e.g. "gzip", or "" for no
// compression.
MediaTypeSuffix() string

// WithOpt applies an option and can be chained.
WithOpt(CompressorOpt) Compressor
}

// CompressorOpt is a compressor option which can be used to configure a
// compressor.
type CompressorOpt interface{}

type noopCompressor struct{}

func (nc noopCompressor) Compress(r io.Reader) (io.ReadCloser, error) {
Expand All @@ -37,16 +44,24 @@ func (nc noopCompressor) MediaTypeSuffix() string {
// NoopCompressor provides no compression.
var NoopCompressor Compressor = noopCompressor{}

func (nc noopCompressor) WithOpt(CompressorOpt) Compressor {
return nc
}

// GzipCompressor provides gzip compression.
var GzipCompressor Compressor = gzipCompressor{}
var GzipCompressor Compressor = gzipCompressor{blockSize: 256 << 10}

type GzipBlockSize int

type gzipCompressor struct{}
type gzipCompressor struct {
blockSize int
}

func (gz gzipCompressor) Compress(reader io.Reader) (io.ReadCloser, error) {
pipeReader, pipeWriter := io.Pipe()

gzw := gzip.NewWriter(pipeWriter)
if err := gzw.SetConcurrency(256<<10, 2*runtime.NumCPU()); err != nil {
if err := gzw.SetConcurrency(gz.blockSize, 2*runtime.NumCPU()); err != nil {
return nil, errors.Wrapf(err, "set concurrency level to %v blocks", 2*runtime.NumCPU())
}
go func() {
Expand Down Expand Up @@ -76,6 +91,15 @@ func (gz gzipCompressor) MediaTypeSuffix() string {
return "gzip"
}

func (gz gzipCompressor) WithOpt(opt CompressorOpt) Compressor {
switch val := opt.(type) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I like that we ignore invalid arguments here. If we make this act on *gzipCompressor we could make it return a bool (or error) and then the caller can decide if they care about whether WithOpt actually did anything.

Alternatively, it feels like it should be possible to reimplement all of this compressor stuff with a closure that is passed by the caller (that creates the writer and configures it). But that would require more work...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we could add a CustomCompressor just for cases where you need to configure the compression? Idk...

case GzipBlockSize:
gz.blockSize = int(val)
}

return gz
}

// ZstdCompressor provides zstd compression.
var ZstdCompressor Compressor = zstdCompressor{}

Expand Down Expand Up @@ -114,3 +138,7 @@ func (zs zstdCompressor) Compress(reader io.Reader) (io.ReadCloser, error) {
func (zs zstdCompressor) MediaTypeSuffix() string {
return "zstd"
}

func (zs zstdCompressor) WithOpt(CompressorOpt) Compressor {
return zs
}
16 changes: 16 additions & 0 deletions mutate/compress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,22 @@ func TestGzipCompressor(t *testing.T) {
assert.NoError(err)

assert.Equal(string(content), fact)

// with options
buf = bytes.NewBufferString(fact)
c = GzipCompressor.WithOpt(GzipBlockSize(256 << 12))

r, err = c.Compress(buf)
assert.NoError(err)
assert.Equal(c.MediaTypeSuffix(), "gzip")

r, err = gzip.NewReader(r)
assert.NoError(err)

content, err = ioutil.ReadAll(r)
assert.NoError(err)

assert.Equal(string(content), fact)
}

func TestZstdCompressor(t *testing.T) {
Expand Down