From 41c8aaad18719ef286e0f5a2fd332ea9a1a89efc Mon Sep 17 00:00:00 2001 From: Adam Shannon Date: Mon, 8 Apr 2024 14:48:03 -0500 Subject: [PATCH] filelist: use buffered channel for consistent listings --- pkg/filelist/bucket.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pkg/filelist/bucket.go b/pkg/filelist/bucket.go index 0349a376..8ae43790 100644 --- a/pkg/filelist/bucket.go +++ b/pkg/filelist/bucket.go @@ -137,7 +137,7 @@ func (ls *bucketLister) listFilesFromGCSBucket(opts ListOpts, pathPrefix string) var g errgroup.Group datePrefixes := yyyymmdd.Prefixes(opts.StartDate, opts.EndDate) - discoveredFiles := make(chan []File) + discoveredFiles := make(chan []File, len(datePrefixes)) for _, datePrefix := range datePrefixes { g.Go(func() error { @@ -172,28 +172,20 @@ func (ls *bucketLister) listFilesFromGCSBucket(opts ListOpts, pathPrefix string) files, err := ls.listFilesFromCursor(opts, ls.buck.List(listOptions)) if len(files) > 0 { - go func() { - discoveredFiles <- files - }() + discoveredFiles <- files } return err }) } err := g.Wait() - go func() { - discoveredFiles <- nil - }() if err != nil { return nil, err } + close(discoveredFiles) var out []File - for { - files := <-discoveredFiles - if len(files) == 0 { - break - } + for files := range discoveredFiles { out = append(out, files...) } return out, nil