diff --git a/bindings/go/scip/memtest/DO_NOT_ADD_NEW_TEST_FILES_HERE b/bindings/go/scip/memtest/DO_NOT_ADD_NEW_TEST_FILES_HERE new file mode 100644 index 00000000..9b303589 --- /dev/null +++ b/bindings/go/scip/memtest/DO_NOT_ADD_NEW_TEST_FILES_HERE @@ -0,0 +1,4 @@ +This package should only have a single test +so that we can use SetMemoryLimit without +affecting other tests running in different +goroutines in the same process. diff --git a/bindings/go/scip/memtest/low_mem_test.go b/bindings/go/scip/memtest/low_mem_test.go new file mode 100644 index 00000000..4d85ffab --- /dev/null +++ b/bindings/go/scip/memtest/low_mem_test.go @@ -0,0 +1,63 @@ +package memtest + +import ( + "fmt" + "io" + "os" + "runtime" + "runtime/debug" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + "github.com/sourcegraph/scip/bindings/go/scip" +) + +// Do not add any other tests in this sub-package, so that the +// SetMemoryLimit call doesn't interfere with other running tests + +func TestLowMemoryParsing(t *testing.T) { + tmpFile, err := os.CreateTemp(t.TempDir(), "very-large-index.scip") + require.NoError(t, err) + defer os.RemoveAll(tmpFile.Name()) + + // Total index size will be about (textSize + ε) * docCount ~= 128 MB + const textSize = 128 * 1024 + const docCount = 1000 + { + largeIndex := scip.Index{} + for i := 0; i < docCount; i++ { + doc := scip.Document{} + doc.Text = strings.Repeat(fmt.Sprintf("%d", i), textSize) + largeIndex.Documents = append(largeIndex.Documents, &doc) + } + indexBytes, err := proto.Marshal(&largeIndex) + require.NoError(t, err) + _, err = tmpFile.Write(indexBytes) + require.NoError(t, err) + + _, err = tmpFile.Seek(0, io.SeekStart) + require.NoError(t, err) + + runtime.GC() + } + + require.Greater(t, docCount, 100) + const maxDocsInMemory = docCount / 100 + debug.SetMemoryLimit(textSize * maxDocsInMemory) + + curDoc := &scip.Document{} + indexVisitor := scip.IndexVisitor{VisitDocument: func(d *scip.Document) { + curDoc = d + }} + + // No OOM + err = indexVisitor.ParseStreaming(tmpFile) + _ = curDoc + require.NoError(t, err) +} + +// Do not add any other tests in this sub-package, so that the +// SetMemoryLimit call doesn't interfere with other running tests