From dd0ee209f93867ea1f7cdf766fe59f1d3b6cf5fc Mon Sep 17 00:00:00 2001 From: obitech Date: Mon, 22 Jul 2019 21:38:01 +0200 Subject: [PATCH 01/13] Primitive printing of block in CLI unit test - Moving necessary functions into different package (only copy) - Output not checked yet Signed-off-by: obitech --- cmd/tsdb/main_test.go | 43 +++++++++++ testutil/db.go | 164 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 cmd/tsdb/main_test.go create mode 100644 testutil/db.go diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go new file mode 100644 index 00000000..52794c1e --- /dev/null +++ b/cmd/tsdb/main_test.go @@ -0,0 +1,43 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/prometheus/tsdb" + "github.com/prometheus/tsdb/testutil" +) + +func TestCLI(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "test") + testutil.Ok(t, err) + defer func() { + testutil.Ok(t, os.RemoveAll(tmpdir)) + }() + + safeDBOptions := *tsdb.DefaultOptions + safeDBOptions.RetentionDuration = 0 + + testutil.CreateBlock(t, tmpdir, testutil.GenSeries(1, 1, 0, 1)) + db, err := tsdb.Open(tmpdir, nil, nil, &safeDBOptions) + defer func() { + testutil.Ok(t, db.Close()) + }() + hr := true + + printBlocks(db.Blocks(), &hr) +} diff --git a/testutil/db.go b/testutil/db.go new file mode 100644 index 00000000..54ad5bc0 --- /dev/null +++ b/testutil/db.go @@ -0,0 +1,164 @@ +package testutil + +import ( + "context" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "sort" + "strconv" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/tsdb" + "github.com/prometheus/tsdb/labels" + "github.com/prometheus/tsdb/tsdbutil" +) + +// OpenTestDB opens a test Database +func OpenTestDB(t testing.TB, opts *tsdb.Options) (db *tsdb.DB, close func()) { + tmpdir, err := ioutil.TempDir("", "test") + Ok(t, err) + + db, err = tsdb.Open(tmpdir, nil, nil, opts) + Ok(t, err) + + // Do not close the test database by default as it will deadlock on test failures. + return db, func() { + Ok(t, os.RemoveAll(tmpdir)) + } +} + +// CreateBlock creates a block with given set of series and returns its dir. +func CreateBlock(tb testing.TB, dir string, series []tsdb.Series) string { + head := createHead(tb, series) + compactor, err := tsdb.NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) + Ok(tb, err) + + Ok(tb, os.MkdirAll(dir, 0777)) + + // Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime). + // Because of this block intervals are always +1 than the total samples it includes. + ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime()+1, nil) + Ok(tb, err) + return filepath.Join(dir, ulid.String()) +} + +func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { + head, err := tsdb.NewHead(nil, nil, nil, 2*60*60*1000) + Ok(tb, err) + defer head.Close() + + app := head.Appender() + for _, s := range series { + ref := uint64(0) + it := s.Iterator() + for it.Next() { + t, v := it.At() + if ref != 0 { + err := app.AddFast(ref, t, v) + if err == nil { + continue + } + } + ref, err = app.Add(s.Labels(), t, v) + Ok(tb, err) + } + Ok(tb, it.Err()) + } + err = app.Commit() + Ok(tb, err) + return head +} + +const ( + defaultLabelName = "labelName" + defaultLabelValue = "labelValue" +) + +type sample struct { + t int64 + v float64 +} + +func (s sample) T() int64 { + return s.t +} + +func (s sample) V() float64 { + return s.v +} + +// GenSeries generates series with a given number of labels and values. +func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { + if totalSeries == 0 || labelCount == 0 { + return nil + } + + series := make([]tsdb.Series, totalSeries) + + for i := 0; i < totalSeries; i++ { + lbls := make(map[string]string, labelCount) + lbls[defaultLabelName] = strconv.Itoa(i) + for j := 1; len(lbls) < labelCount; j++ { + lbls[defaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) + } + samples := make([]tsdbutil.Sample, 0, maxt-mint+1) + for t := mint; t < maxt; t++ { + samples = append(samples, sample{t: t, v: rand.Float64()}) + } + series[i] = newSeries(lbls, samples) + } + return series +} + +type mockSeries struct { + labels func() labels.Labels + iterator func() tsdb.SeriesIterator +} + +func newSeries(l map[string]string, s []tsdbutil.Sample) tsdb.Series { + return &mockSeries{ + labels: func() labels.Labels { return labels.FromMap(l) }, + iterator: func() tsdb.SeriesIterator { return newListSeriesIterator(s) }, + } +} +func (m *mockSeries) Labels() labels.Labels { return m.labels() } +func (m *mockSeries) Iterator() tsdb.SeriesIterator { return m.iterator() } + +type listSeriesIterator struct { + list []tsdbutil.Sample + idx int +} + +func newListSeriesIterator(list []tsdbutil.Sample) *listSeriesIterator { + return &listSeriesIterator{list: list, idx: -1} +} + +func (it *listSeriesIterator) At() (int64, float64) { + s := it.list[it.idx] + return s.T(), s.V() +} + +func (it *listSeriesIterator) Next() bool { + it.idx++ + return it.idx < len(it.list) +} + +func (it *listSeriesIterator) Seek(t int64) bool { + if it.idx == -1 { + it.idx = 0 + } + // Do binary search between current position and end. + it.idx = sort.Search(len(it.list)-it.idx, func(i int) bool { + s := it.list[i+it.idx] + return s.T() >= t + }) + + return it.idx < len(it.list) +} + +func (it *listSeriesIterator) Err() error { + return nil +} From 0957af6b799cc568121c21a988542daae524973d Mon Sep 17 00:00:00 2001 From: obitech Date: Tue, 23 Jul 2019 22:25:00 +0200 Subject: [PATCH 02/13] Setup Example output test Signed-off-by: obitech --- cmd/tsdb/main_test.go | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 52794c1e..a7ae0d82 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -16,28 +16,40 @@ package main import ( "io/ioutil" "os" - "testing" "github.com/prometheus/tsdb" "github.com/prometheus/tsdb/testutil" ) -func TestCLI(t *testing.T) { +func createTestDBWithBlock() (db *tsdb.DB, close func()) { tmpdir, err := ioutil.TempDir("", "test") - testutil.Ok(t, err) - defer func() { - testutil.Ok(t, os.RemoveAll(tmpdir)) - }() + if err != nil { + os.Exit(1) + } safeDBOptions := *tsdb.DefaultOptions safeDBOptions.RetentionDuration = 0 - testutil.CreateBlock(t, tmpdir, testutil.GenSeries(1, 1, 0, 1)) - db, err := tsdb.Open(tmpdir, nil, nil, &safeDBOptions) + testutil.CreateBlock(nil, tmpdir, testutil.GenSeries(1, 1, 0, 1)) + db, err = tsdb.Open(tmpdir, nil, nil, &safeDBOptions) + if err != nil { + os.RemoveAll(tmpdir) + os.Exit(1) + } + return db, func() { + db.Close() + os.RemoveAll(tmpdir) + } +} + +func ExampleCLICalls() { + db, close := createTestDBWithBlock() defer func() { - testutil.Ok(t, db.Close()) + close() }() - hr := true + hr := false printBlocks(db.Blocks(), &hr) + // Output: + // BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES } From 4cad92c0f29e813336066b555e0ac5b2932f5d75 Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 4 Aug 2019 15:32:39 +0200 Subject: [PATCH 03/13] Let printBlocks() take a writer for better testability, write unit test Signed-off-by: obitech --- cmd/tsdb/main.go | 12 +++++--- cmd/tsdb/main_test.go | 71 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/cmd/tsdb/main.go b/cmd/tsdb/main.go index e3dc530a..8eb98176 100644 --- a/cmd/tsdb/main.go +++ b/cmd/tsdb/main.go @@ -39,6 +39,10 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) +const ( + printBlocksTableHeader = "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES" +) + func main() { if err := execute(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -95,7 +99,7 @@ func execute() (err error) { if err != nil { return err } - printBlocks(blocks, listCmdHumanReadable) + printBlocks(os.Stdout, blocks, listCmdHumanReadable) case analyzeCmd.FullCommand(): db, err := tsdb.OpenDBReadOnly(*analyzePath, nil) if err != nil { @@ -434,11 +438,11 @@ func readPrometheusLabels(r io.Reader, n int) ([]labels.Labels, error) { return mets, nil } -func printBlocks(blocks []tsdb.BlockReader, humanReadable *bool) { - tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) +func printBlocks(w io.Writer, blocks []tsdb.BlockReader, humanReadable *bool) { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) defer tw.Flush() - fmt.Fprintln(tw, "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES") + fmt.Fprintln(tw, printBlocksTableHeader) for _, b := range blocks { meta := b.Meta() diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index a7ae0d82..3f00fe9f 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -14,17 +14,23 @@ package main import ( + "bytes" + "fmt" "io/ioutil" "os" + "strings" + "testing" + "text/tabwriter" "github.com/prometheus/tsdb" "github.com/prometheus/tsdb/testutil" ) -func createTestDBWithBlock() (db *tsdb.DB, close func()) { +func createTestDBWithBlock(t *testing.T) (db *tsdb.DB, close func()) { tmpdir, err := ioutil.TempDir("", "test") if err != nil { - os.Exit(1) + os.RemoveAll(tmpdir) + t.Error(err) } safeDBOptions := *tsdb.DefaultOptions @@ -34,7 +40,7 @@ func createTestDBWithBlock() (db *tsdb.DB, close func()) { db, err = tsdb.Open(tmpdir, nil, nil, &safeDBOptions) if err != nil { os.RemoveAll(tmpdir) - os.Exit(1) + t.Error(err) } return db, func() { db.Close() @@ -42,14 +48,57 @@ func createTestDBWithBlock() (db *tsdb.DB, close func()) { } } -func ExampleCLICalls() { - db, close := createTestDBWithBlock() - defer func() { - close() - }() +func TestCLIPrintBlocks(t *testing.T) { + db, closeFn := createTestDBWithBlock(t) + defer closeFn() + var b bytes.Buffer hr := false - printBlocks(db.Blocks(), &hr) - // Output: - // BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES + tw := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0) + defer tw.Flush() + + // Set table header + _, err := fmt.Fprintln(&b, printBlocksTableHeader) + if err != nil { + t.Error(err) + } + + // Test table header + actual := b.String() + expected := fmt.Sprintln(printBlocksTableHeader) + if expected != actual { + t.Errorf("expected (%#v) != actual (%#v)", expected, actual) + } + + // Set table contents + _, err = fmt.Fprintf(&b, + "%v\t%v\t%v\t%v\t%v\t%v\n", + db.Blocks()[0].Meta().ULID, + getFormatedTime(db.Blocks()[0].Meta().MinTime, &hr), + getFormatedTime(db.Blocks()[0].Meta().MaxTime, &hr), + db.Blocks()[0].Meta().Stats.NumSamples, + db.Blocks()[0].Meta().Stats.NumChunks, + db.Blocks()[0].Meta().Stats.NumSeries, + ) + if err != nil { + t.Error(err) + } + + // Test table contents + var actualStdout bytes.Buffer + printBlocks(&actualStdout, db.Blocks(), &hr) + + actual = actualStdout.String() + actual = strings.Replace(actual, " ", "", -1) + actual = strings.Replace(actual, "\t", "", -1) + actual = strings.Replace(actual, "\n", "", -1) + + expected = b.String() + expected = strings.Replace(expected, " ", "", -1) + expected = strings.Replace(expected, "\t", "", -1) + expected = strings.Replace(expected, "\n", "", -1) + + if expected != actual { + t.Errorf("expected (%#v) != actual (%#v)", expected, actual) + } } From c18ea5e8d3b993582b4e4697238303a0d79a1171 Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 4 Aug 2019 15:59:40 +0200 Subject: [PATCH 04/13] Adjust syntax according to newer commits Signed-off-by: obitech --- cmd/tsdb/main_test.go | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 3f00fe9f..c6a28ad4 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -26,7 +26,7 @@ import ( "github.com/prometheus/tsdb/testutil" ) -func createTestDBWithBlock(t *testing.T) (db *tsdb.DB, close func()) { +func createTestRODBWithBlock(t *testing.T) (*tsdb.DBReadOnly, func()) { tmpdir, err := ioutil.TempDir("", "test") if err != nil { os.RemoveAll(tmpdir) @@ -37,19 +37,27 @@ func createTestDBWithBlock(t *testing.T) (db *tsdb.DB, close func()) { safeDBOptions.RetentionDuration = 0 testutil.CreateBlock(nil, tmpdir, testutil.GenSeries(1, 1, 0, 1)) - db, err = tsdb.Open(tmpdir, nil, nil, &safeDBOptions) + db, err := tsdb.Open(tmpdir, nil, nil, &safeDBOptions) if err != nil { os.RemoveAll(tmpdir) t.Error(err) } - return db, func() { - db.Close() + if err = db.Close(); err != nil { + t.Error(err) + } + + dbRO, err := tsdb.OpenDBReadOnly(tmpdir, nil) + if err != nil { + t.Error(err) + } + + return dbRO, func() { os.RemoveAll(tmpdir) } } func TestCLIPrintBlocks(t *testing.T) { - db, closeFn := createTestDBWithBlock(t) + db, closeFn := createTestRODBWithBlock(t) defer closeFn() var b bytes.Buffer @@ -71,14 +79,20 @@ func TestCLIPrintBlocks(t *testing.T) { } // Set table contents + blocks, err := db.Blocks() + if err != nil { + t.Error(err) + } + meta := blocks[0].Meta() + _, err = fmt.Fprintf(&b, "%v\t%v\t%v\t%v\t%v\t%v\n", - db.Blocks()[0].Meta().ULID, - getFormatedTime(db.Blocks()[0].Meta().MinTime, &hr), - getFormatedTime(db.Blocks()[0].Meta().MaxTime, &hr), - db.Blocks()[0].Meta().Stats.NumSamples, - db.Blocks()[0].Meta().Stats.NumChunks, - db.Blocks()[0].Meta().Stats.NumSeries, + meta.ULID, + getFormatedTime(meta.MinTime, &hr), + getFormatedTime(meta.MaxTime, &hr), + meta.Stats.NumSamples, + meta.Stats.NumChunks, + meta.Stats.NumSeries, ) if err != nil { t.Error(err) @@ -86,7 +100,11 @@ func TestCLIPrintBlocks(t *testing.T) { // Test table contents var actualStdout bytes.Buffer - printBlocks(&actualStdout, db.Blocks(), &hr) + blocks, err = db.Blocks() + if err != nil { + t.Error(err) + } + printBlocks(&actualStdout, blocks, &hr) actual = actualStdout.String() actual = strings.Replace(actual, " ", "", -1) From 58712f9c32509ed9408b791049125c17dcde6f1f Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 4 Aug 2019 16:24:35 +0200 Subject: [PATCH 05/13] Fix gofmt in main_test Signed-off-by: obitech --- cmd/tsdb/main_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index c6a28ad4..3e43fbd7 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -93,7 +93,8 @@ func TestCLIPrintBlocks(t *testing.T) { meta.Stats.NumSamples, meta.Stats.NumChunks, meta.Stats.NumSeries, - ) + ) + if err != nil { t.Error(err) } From eebcb60f3f01c60c68e9f1459766a4562938d32a Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 4 Aug 2019 18:23:14 +0200 Subject: [PATCH 06/13] Fix circular import Signed-off-by: obitech --- cmd/tsdb/main_test.go | 6 +++--- testutil/{ => db}/db.go | 46 ++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 26 deletions(-) rename testutil/{ => db}/db.go (80%) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 3e43fbd7..4b9d3927 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright 2019 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,7 +23,7 @@ import ( "text/tabwriter" "github.com/prometheus/tsdb" - "github.com/prometheus/tsdb/testutil" + testutildb "github.com/prometheus/tsdb/testutil/db" ) func createTestRODBWithBlock(t *testing.T) (*tsdb.DBReadOnly, func()) { @@ -36,7 +36,7 @@ func createTestRODBWithBlock(t *testing.T) (*tsdb.DBReadOnly, func()) { safeDBOptions := *tsdb.DefaultOptions safeDBOptions.RetentionDuration = 0 - testutil.CreateBlock(nil, tmpdir, testutil.GenSeries(1, 1, 0, 1)) + testutildb.CreateBlock(nil, tmpdir, testutildb.GenSeries(1, 1, 0, 1)) db, err := tsdb.Open(tmpdir, nil, nil, &safeDBOptions) if err != nil { os.RemoveAll(tmpdir) diff --git a/testutil/db.go b/testutil/db/db.go similarity index 80% rename from testutil/db.go rename to testutil/db/db.go index 54ad5bc0..52311ecd 100644 --- a/testutil/db.go +++ b/testutil/db/db.go @@ -1,8 +1,20 @@ -package testutil +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db import ( "context" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -11,43 +23,31 @@ import ( "testing" "github.com/go-kit/kit/log" + "github.com/prometheus/tsdb" "github.com/prometheus/tsdb/labels" + "github.com/prometheus/tsdb/testutil" "github.com/prometheus/tsdb/tsdbutil" ) -// OpenTestDB opens a test Database -func OpenTestDB(t testing.TB, opts *tsdb.Options) (db *tsdb.DB, close func()) { - tmpdir, err := ioutil.TempDir("", "test") - Ok(t, err) - - db, err = tsdb.Open(tmpdir, nil, nil, opts) - Ok(t, err) - - // Do not close the test database by default as it will deadlock on test failures. - return db, func() { - Ok(t, os.RemoveAll(tmpdir)) - } -} - // CreateBlock creates a block with given set of series and returns its dir. func CreateBlock(tb testing.TB, dir string, series []tsdb.Series) string { head := createHead(tb, series) compactor, err := tsdb.NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) - Ok(tb, err) + testutil.Ok(tb, err) - Ok(tb, os.MkdirAll(dir, 0777)) + testutil.Ok(tb, os.MkdirAll(dir, 0777)) // Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime). // Because of this block intervals are always +1 than the total samples it includes. ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime()+1, nil) - Ok(tb, err) + testutil.Ok(tb, err) return filepath.Join(dir, ulid.String()) } func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { head, err := tsdb.NewHead(nil, nil, nil, 2*60*60*1000) - Ok(tb, err) + testutil.Ok(tb, err) defer head.Close() app := head.Appender() @@ -63,12 +63,12 @@ func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { } } ref, err = app.Add(s.Labels(), t, v) - Ok(tb, err) + testutil.Ok(tb, err) } - Ok(tb, it.Err()) + testutil.Ok(tb, it.Err()) } err = app.Commit() - Ok(tb, err) + testutil.Ok(tb, err) return head } From e5c9d5367d67bdfb5d5fb6b02a6e973ffd577688 Mon Sep 17 00:00:00 2001 From: Alexander Knipping Date: Fri, 9 Aug 2019 17:06:16 +0200 Subject: [PATCH 07/13] Apply suggestions from code review Co-Authored-By: Krasi Georgiev <8903888+krasi-georgiev@users.noreply.github.com> Signed-off-by: obitech --- cmd/tsdb/main_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 4b9d3927..8dedc33c 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -26,7 +26,7 @@ import ( testutildb "github.com/prometheus/tsdb/testutil/db" ) -func createTestRODBWithBlock(t *testing.T) (*tsdb.DBReadOnly, func()) { +func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { tmpdir, err := ioutil.TempDir("", "test") if err != nil { os.RemoveAll(tmpdir) @@ -56,7 +56,7 @@ func createTestRODBWithBlock(t *testing.T) (*tsdb.DBReadOnly, func()) { } } -func TestCLIPrintBlocks(t *testing.T) { +func TestPrintBlocks(t *testing.T) { db, closeFn := createTestRODBWithBlock(t) defer closeFn() @@ -65,7 +65,7 @@ func TestCLIPrintBlocks(t *testing.T) { tw := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0) defer tw.Flush() - // Set table header + // Set table header. _, err := fmt.Fprintln(&b, printBlocksTableHeader) if err != nil { t.Error(err) From 661fa1ce2835f46cd45dcbcf515e6bb0c68fdea9 Mon Sep 17 00:00:00 2001 From: obitech Date: Fri, 9 Aug 2019 17:21:34 +0200 Subject: [PATCH 08/13] Remove uncessary db.Open(), add full stops in comments Signed-off-by: obitech --- cmd/tsdb/main_test.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 8dedc33c..5efd4f8c 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -37,14 +37,6 @@ func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { safeDBOptions.RetentionDuration = 0 testutildb.CreateBlock(nil, tmpdir, testutildb.GenSeries(1, 1, 0, 1)) - db, err := tsdb.Open(tmpdir, nil, nil, &safeDBOptions) - if err != nil { - os.RemoveAll(tmpdir) - t.Error(err) - } - if err = db.Close(); err != nil { - t.Error(err) - } dbRO, err := tsdb.OpenDBReadOnly(tmpdir, nil) if err != nil { @@ -57,7 +49,7 @@ func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { } func TestPrintBlocks(t *testing.T) { - db, closeFn := createTestRODBWithBlock(t) + db, closeFn := createRoDb(t) defer closeFn() var b bytes.Buffer @@ -71,14 +63,14 @@ func TestPrintBlocks(t *testing.T) { t.Error(err) } - // Test table header + // Test table header. actual := b.String() expected := fmt.Sprintln(printBlocksTableHeader) if expected != actual { t.Errorf("expected (%#v) != actual (%#v)", expected, actual) } - // Set table contents + // Set table contents. blocks, err := db.Blocks() if err != nil { t.Error(err) @@ -99,7 +91,7 @@ func TestPrintBlocks(t *testing.T) { t.Error(err) } - // Test table contents + // Test table contents. var actualStdout bytes.Buffer blocks, err = db.Blocks() if err != nil { From 7a61b293f5a5f04f706a0ee57deee9c18185ab7b Mon Sep 17 00:00:00 2001 From: obitech Date: Fri, 9 Aug 2019 18:05:40 +0200 Subject: [PATCH 09/13] Better testability of analyzeBlock command, start tests - Factor out `extractBlock()` to retrieve specific block from db.Blocks() - Write tests for said function - Simple, non-output test for `analyzeBlock()` command Signed-off-by: obitech --- cmd/tsdb/main.go | 69 ++++++++++++++++++++++----------------- cmd/tsdb/main_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 30 deletions(-) diff --git a/cmd/tsdb/main.go b/cmd/tsdb/main.go index 8eb98176..d4a95519 100644 --- a/cmd/tsdb/main.go +++ b/cmd/tsdb/main.go @@ -41,6 +41,7 @@ import ( const ( printBlocksTableHeader = "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES" + defaultAnalyzeLimit = "20" ) func main() { @@ -66,7 +67,7 @@ func execute() (err error) { analyzeCmd = cli.Command("analyze", "analyze churn, label pair cardinality.") analyzePath = analyzeCmd.Arg("db path", "database path (default is "+defaultDBPath+")").Default(defaultDBPath).String() analyzeBlockID = analyzeCmd.Arg("block id", "block to analyze (default is the last block)").String() - analyzeLimit = analyzeCmd.Flag("limit", "how many items to show in each list").Default("20").Int() + analyzeLimit = analyzeCmd.Flag("limit", "how many items to show in each list").Default(defaultAnalyzeLimit).Int() dumpCmd = cli.Command("dump", "dump samples from a TSDB") dumpPath = dumpCmd.Arg("db path", "database path (default is "+defaultDBPath+")").Default(defaultDBPath).String() dumpMinTime = dumpCmd.Flag("min-time", "minimum timestamp to dump").Default(strconv.FormatInt(math.MinInt64, 10)).Int64() @@ -114,21 +115,12 @@ func execute() (err error) { if err != nil { return err } - var block tsdb.BlockReader - if *analyzeBlockID != "" { - for _, b := range blocks { - if b.Meta().ULID.String() == *analyzeBlockID { - block = b - break - } - } - } else if len(blocks) > 0 { - block = blocks[len(blocks)-1] - } - if block == nil { - return fmt.Errorf("block not found") + block, err := extractBlock(blocks, analyzeBlockID) + if err != nil { + return err } - return analyzeBlock(block, *analyzeLimit) + + return analyzeBlock(os.Stdout, block, *analyzeLimit) case dumpCmd.FullCommand(): db, err := tsdb.OpenDBReadOnly(*dumpPath, nil) if err != nil { @@ -144,6 +136,25 @@ func execute() (err error) { return nil } +// extractBlock takes a slice of BlockReader and returns a specific block by ID. +func extractBlock(blocks []tsdb.BlockReader, analyzeBlockID *string) (tsdb.BlockReader, error) { + var block tsdb.BlockReader + if *analyzeBlockID != "" { + for _, b := range blocks { + if b.Meta().ULID.String() == *analyzeBlockID { + block = b + break + } + } + } else if len(blocks) > 0 { + block = blocks[len(blocks)-1] + } + if block == nil { + return nil, fmt.Errorf("block not found") + } + return block, nil +} + type writeBenchmark struct { outPath string samplesFile string @@ -465,12 +476,12 @@ func getFormatedTime(timestamp int64, humanReadable *bool) string { return strconv.FormatInt(timestamp, 10) } -func analyzeBlock(b tsdb.BlockReader, limit int) error { +func analyzeBlock(w io.Writer, b tsdb.BlockReader, limit int) error { meta := b.Meta() - fmt.Printf("Block ID: %s\n", meta.ULID) + fmt.Fprintf(w, "Block ID: %s\n", meta.ULID) // Presume 1ms resolution that Prometheus uses. - fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String()) - fmt.Printf("Series: %d\n", meta.Stats.NumSeries) + fmt.Fprintf(w, "Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String()) + fmt.Fprintf(w, "Series: %d\n", meta.Stats.NumSeries) ir, err := b.Index() if err != nil { return err @@ -481,7 +492,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { if err != nil { return err } - fmt.Printf("Label names: %d\n", len(allLabelNames)) + fmt.Fprintf(w, "Label names: %d\n", len(allLabelNames)) type postingInfo struct { key string @@ -493,7 +504,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { sort.Slice(postingInfos, func(i, j int) bool { return postingInfos[i].metric > postingInfos[j].metric }) for i, pc := range postingInfos { - fmt.Printf("%d %s\n", pc.metric, pc.key) + fmt.Fprintf(w, "%d %s\n", pc.metric, pc.key) if i >= limit { break } @@ -527,15 +538,15 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { if p.Err() != nil { return p.Err() } - fmt.Printf("Postings (unique label pairs): %d\n", len(labelpairsUncovered)) - fmt.Printf("Postings entries (total label pairs): %d\n", entries) + fmt.Fprintf(w, "Postings (unique label pairs): %d\n", len(labelpairsUncovered)) + fmt.Fprintf(w, "Postings entries (total label pairs): %d\n", entries) postingInfos = postingInfos[:0] for k, m := range labelpairsUncovered { postingInfos = append(postingInfos, postingInfo{k, uint64(float64(m) / float64(meta.MaxTime-meta.MinTime))}) } - fmt.Printf("\nLabel pairs most involved in churning:\n") + fmt.Fprintf(w, "\nLabel pairs most involved in churning:\n") printInfo(postingInfos) postingInfos = postingInfos[:0] @@ -543,7 +554,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { postingInfos = append(postingInfos, postingInfo{k, uint64(float64(m) / float64(meta.MaxTime-meta.MinTime))}) } - fmt.Printf("\nLabel names most involved in churning:\n") + fmt.Fprintf(w, "\nLabel names most involved in churning:\n") printInfo(postingInfos) postingInfos = postingInfos[:0] @@ -551,7 +562,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { postingInfos = append(postingInfos, postingInfo{k, m}) } - fmt.Printf("\nMost common label pairs:\n") + fmt.Fprintf(w, "\nMost common label pairs:\n") printInfo(postingInfos) postingInfos = postingInfos[:0] @@ -575,7 +586,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { postingInfos = append(postingInfos, postingInfo{n, cumulativeLength}) } - fmt.Printf("\nLabel names with highest cumulative label value length:\n") + fmt.Fprintf(w, "\nLabel names with highest cumulative label value length:\n") printInfo(postingInfos) postingInfos = postingInfos[:0] @@ -586,7 +597,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { } postingInfos = append(postingInfos, postingInfo{n, uint64(lv.Len())}) } - fmt.Printf("\nHighest cardinality labels:\n") + fmt.Fprintf(w, "\nHighest cardinality labels:\n") printInfo(postingInfos) postingInfos = postingInfos[:0] @@ -614,7 +625,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error { postingInfos = append(postingInfos, postingInfo{n, uint64(count)}) } } - fmt.Printf("\nHighest cardinality metric names:\n") + fmt.Fprintf(w, "\nHighest cardinality metric names:\n") printInfo(postingInfos) return nil } diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 5efd4f8c..0c1f03ac 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "os" + "strconv" "strings" "testing" "text/tabwriter" @@ -92,11 +93,12 @@ func TestPrintBlocks(t *testing.T) { } // Test table contents. - var actualStdout bytes.Buffer blocks, err = db.Blocks() if err != nil { t.Error(err) } + + var actualStdout bytes.Buffer printBlocks(&actualStdout, blocks, &hr) actual = actualStdout.String() @@ -113,3 +115,74 @@ func TestPrintBlocks(t *testing.T) { t.Errorf("expected (%#v) != actual (%#v)", expected, actual) } } + +func TestExtractBlock(t *testing.T) { + db, closeFn := createRoDb(t) + defer closeFn() + + blocks, err := db.Blocks() + if err != nil { + t.Error(err) + } + + var analyzeBlockID string + + // Pass: analyze last block (default). + block, err := extractBlock(blocks, &analyzeBlockID) + if err != nil { + t.Error(err) + } + if block == nil { + t.Error("block shouldn't be nil") + } + + // Pass: analyze specific block. + analyzeBlockID = block.Meta().ULID.String() + block, err = extractBlock(blocks, &analyzeBlockID) + if err != nil { + t.Error(err) + } + if block == nil { + t.Error("block shouldn't be nil") + } + + // Fail: analyze non-existing block + analyzeBlockID = "foo" + block, err = extractBlock(blocks, &analyzeBlockID) + if err == nil { + t.Errorf("Analyzing block %q should throw error", analyzeBlockID) + } + if block != nil { + t.Error("block should be nil") + } +} + +func TestAnalyzeBlocks(t *testing.T) { + db, closeFn := createRoDb(t) + defer closeFn() + + blocks, err := db.Blocks() + if err != nil { + t.Error(err) + } + + var analyzeBlockID string + block, err := extractBlock(blocks, &analyzeBlockID) + if err != nil { + t.Error(err) + } + if block == nil { + t.Errorf("block shouldn't be nil") + } + + dal, err := strconv.Atoi(defaultAnalyzeLimit) + if err != nil { + t.Error(err) + } + + var actual bytes.Buffer + err = analyzeBlock(&actual, block, dal) + if err != nil { + t.Error(err) + } +} From 67c92d82729b2f5d78c9e7123685c30cfb44fc60 Mon Sep 17 00:00:00 2001 From: obitech Date: Fri, 9 Aug 2019 21:40:04 +0200 Subject: [PATCH 10/13] Add tests for analyzeBlock CLI command Signed-off-by: obitech --- cmd/tsdb/main.go | 6 +++--- cmd/tsdb/main_test.go | 46 ++++++++++++++++++++++++++++++++++++--- testutil/db/db.go | 50 +++++++++++++++++++++---------------------- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/cmd/tsdb/main.go b/cmd/tsdb/main.go index d4a95519..23a72d64 100644 --- a/cmd/tsdb/main.go +++ b/cmd/tsdb/main.go @@ -32,16 +32,18 @@ import ( "github.com/go-kit/kit/log" "github.com/pkg/errors" + "gopkg.in/alecthomas/kingpin.v2" + "github.com/prometheus/tsdb" "github.com/prometheus/tsdb/chunks" tsdb_errors "github.com/prometheus/tsdb/errors" "github.com/prometheus/tsdb/labels" - "gopkg.in/alecthomas/kingpin.v2" ) const ( printBlocksTableHeader = "BLOCK ULID\tMIN TIME\tMAX TIME\tNUM SAMPLES\tNUM CHUNKS\tNUM SERIES" defaultAnalyzeLimit = "20" + timeDelta = 30000 ) func main() { @@ -250,8 +252,6 @@ func (b *writeBenchmark) run() error { return nil } -const timeDelta = 30000 - func (b *writeBenchmark) ingestScrapes(lbls []labels.Labels, scrapeCount int) (uint64, error) { var mu sync.Mutex var total uint64 diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 0c1f03ac..33b9f297 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" "text/tabwriter" + "time" "github.com/prometheus/tsdb" testutildb "github.com/prometheus/tsdb/testutil/db" @@ -112,7 +113,7 @@ func TestPrintBlocks(t *testing.T) { expected = strings.Replace(expected, "\n", "", -1) if expected != actual { - t.Errorf("expected (%#v) != actual (%#v)", expected, actual) + t.Errorf("expected (%#v) != actual (%#v)", b.String(), actualStdout.String()) } } @@ -150,7 +151,7 @@ func TestExtractBlock(t *testing.T) { analyzeBlockID = "foo" block, err = extractBlock(blocks, &analyzeBlockID) if err == nil { - t.Errorf("Analyzing block %q should throw error", analyzeBlockID) + t.Errorf("Analyzing block ID %q should throw error", analyzeBlockID) } if block != nil { t.Error("block should be nil") @@ -180,9 +181,48 @@ func TestAnalyzeBlocks(t *testing.T) { t.Error(err) } - var actual bytes.Buffer + var ( + expected bytes.Buffer + actual bytes.Buffer + ) + + // Actual output. err = analyzeBlock(&actual, block, dal) if err != nil { t.Error(err) } + + act := actual.String() + act = strings.Replace(act, " ", "", -1) + act = strings.Replace(act, "\t", "", -1) + act = strings.Replace(act, "\n", "", -1) + + // Expected output. + meta := block.Meta() + fmt.Fprintf(&expected, "Block ID: %s\n", meta.ULID) + fmt.Fprintf(&expected, "Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String()) + fmt.Fprintf(&expected, "Series: %d\n", 1) + fmt.Fprintf(&expected, "Label names: %d\n", 1) + fmt.Fprintf(&expected, "Postings (unique label pairs): %d\n", 1) + fmt.Fprintf(&expected, "Postings entries (total label pairs): %d\n", 1) + fmt.Fprintf(&expected, "\nLabel pairs most involved in churning:\n") + fmt.Fprintf(&expected, "1 %s=0", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "\nLabel names most involved in churning:\n") + fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "\nMost common label pairs:\n") + fmt.Fprintf(&expected, "1 %s=0", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "\nLabel names with highest cumulative label value length:\n") + fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "\nHighest cardinality labels:\n") + fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "\nHighest cardinality metric names:\n") + + exp := expected.String() + exp = strings.Replace(exp, " ", "", -1) + exp = strings.Replace(exp, "\t", "", -1) + exp = strings.Replace(exp, "\n", "", -1) + + if exp != act { + t.Errorf("expected (%#v) != actual (%#v)", expected.String(), actual.String()) + } } diff --git a/testutil/db/db.go b/testutil/db/db.go index 52311ecd..11a5ad41 100644 --- a/testutil/db/db.go +++ b/testutil/db/db.go @@ -30,6 +30,29 @@ import ( "github.com/prometheus/tsdb/tsdbutil" ) +const ( + DefaultLabelName = "labelName" + defaultLabelValue = "labelValue" +) + +type sample struct { + t int64 + v float64 +} + +type mockSeries struct { + labels func() labels.Labels + iterator func() tsdb.SeriesIterator +} + +func (s sample) T() int64 { + return s.t +} + +func (s sample) V() float64 { + return s.v +} + // CreateBlock creates a block with given set of series and returns its dir. func CreateBlock(tb testing.TB, dir string, series []tsdb.Series) string { head := createHead(tb, series) @@ -72,24 +95,6 @@ func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { return head } -const ( - defaultLabelName = "labelName" - defaultLabelValue = "labelValue" -) - -type sample struct { - t int64 - v float64 -} - -func (s sample) T() int64 { - return s.t -} - -func (s sample) V() float64 { - return s.v -} - // GenSeries generates series with a given number of labels and values. func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { if totalSeries == 0 || labelCount == 0 { @@ -100,9 +105,9 @@ func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { for i := 0; i < totalSeries; i++ { lbls := make(map[string]string, labelCount) - lbls[defaultLabelName] = strconv.Itoa(i) + lbls[DefaultLabelName] = strconv.Itoa(i) for j := 1; len(lbls) < labelCount; j++ { - lbls[defaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) + lbls[DefaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) } samples := make([]tsdbutil.Sample, 0, maxt-mint+1) for t := mint; t < maxt; t++ { @@ -113,11 +118,6 @@ func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { return series } -type mockSeries struct { - labels func() labels.Labels - iterator func() tsdb.SeriesIterator -} - func newSeries(l map[string]string, s []tsdbutil.Sample) tsdb.Series { return &mockSeries{ labels: func() labels.Labels { return labels.FromMap(l) }, From 4dd7e7f7ee12d90d64ccca3848f40679a71810ec Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 11 Aug 2019 12:23:43 +0200 Subject: [PATCH 11/13] Move testing functions back into tsdb package, remove duplicates from existing test files Signed-off-by: obitech --- block_test.go | 81 +++------------------------------- cmd/tsdb/main_test.go | 13 +++--- compact_test.go | 12 ++--- db_test.go | 24 +++++----- head_test.go | 2 +- testutil/db/db.go => mocks.go | 83 ++++++++++++++++------------------- querier_test.go | 70 +++++------------------------ 7 files changed, 78 insertions(+), 207 deletions(-) rename testutil/db/db.go => mocks.go (56%) diff --git a/block_test.go b/block_test.go index 3f39a899..490ef149 100644 --- a/block_test.go +++ b/block_test.go @@ -16,16 +16,15 @@ package tsdb import ( "context" "encoding/binary" - "errors" "io/ioutil" "math/rand" "os" "path/filepath" - "strconv" "testing" "github.com/go-kit/kit/log" + "github.com/prometheus/tsdb/chunks" "github.com/prometheus/tsdb/labels" "github.com/prometheus/tsdb/testutil" @@ -57,7 +56,7 @@ func TestSetCompactionFailed(t *testing.T) { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 1)) + blockDir := MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 1)) b, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) testutil.Equals(t, false, b.meta.Compaction.Failed) @@ -77,7 +76,7 @@ func TestCreateBlock(t *testing.T) { defer func() { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - b, err := OpenBlock(nil, createBlock(t, tmpdir, genSeries(1, 1, 0, 10)), nil) + b, err := OpenBlock(nil, MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 10)), nil) if err == nil { testutil.Ok(t, b.Close()) } @@ -134,7 +133,7 @@ func TestCorruptedChunk(t *testing.T) { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - blockDir := createBlock(t, tmpdir, genSeries(1, 1, 0, 1)) + blockDir := MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 1)) files, err := sequenceFiles(chunkDir(blockDir)) testutil.Ok(t, err) testutil.Assert(t, len(files) > 0, "No chunk created.") @@ -168,7 +167,7 @@ func TestBlockSize(t *testing.T) { // Create a block and compare the reported size vs actual disk size. { - blockDirInit = createBlock(t, tmpdir, genSeries(10, 1, 1, 100)) + blockDirInit = MockCreateBlock(t, tmpdir, MockGenSeries(10, 1, 1, 100)) blockInit, err = OpenBlock(nil, blockDirInit, nil) testutil.Ok(t, err) defer func() { @@ -204,76 +203,6 @@ func TestBlockSize(t *testing.T) { } } -// createBlock creates a block with given set of series and returns its dir. -func createBlock(tb testing.TB, dir string, series []Series) string { - head := createHead(tb, series) - compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) - testutil.Ok(tb, err) - - testutil.Ok(tb, os.MkdirAll(dir, 0777)) - - // Add +1 millisecond to block maxt because block intervals are half-open: [b.MinTime, b.MaxTime). - // Because of this block intervals are always +1 than the total samples it includes. - ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime()+1, nil) - testutil.Ok(tb, err) - return filepath.Join(dir, ulid.String()) -} - -func createHead(tb testing.TB, series []Series) *Head { - head, err := NewHead(nil, nil, nil, 2*60*60*1000) - testutil.Ok(tb, err) - defer head.Close() - - app := head.Appender() - for _, s := range series { - ref := uint64(0) - it := s.Iterator() - for it.Next() { - t, v := it.At() - if ref != 0 { - err := app.AddFast(ref, t, v) - if err == nil { - continue - } - } - ref, err = app.Add(s.Labels(), t, v) - testutil.Ok(tb, err) - } - testutil.Ok(tb, it.Err()) - } - err = app.Commit() - testutil.Ok(tb, err) - return head -} - -const ( - defaultLabelName = "labelName" - defaultLabelValue = "labelValue" -) - -// genSeries generates series with a given number of labels and values. -func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series { - if totalSeries == 0 || labelCount == 0 { - return nil - } - - series := make([]Series, totalSeries) - - for i := 0; i < totalSeries; i++ { - lbls := make(map[string]string, labelCount) - lbls[defaultLabelName] = strconv.Itoa(i) - for j := 1; len(lbls) < labelCount; j++ { - lbls[defaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) - } - samples := make([]tsdbutil.Sample, 0, maxt-mint+1) - for t := mint; t < maxt; t++ { - samples = append(samples, sample{t: t, v: rand.Float64()}) - } - series[i] = newSeries(lbls, samples) - } - return series -} - // populateSeries generates series from given labels, mint and maxt. func populateSeries(lbls []map[string]string, mint, maxt int64) []Series { if len(lbls) == 0 { diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 33b9f297..543f664c 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -25,7 +25,6 @@ import ( "time" "github.com/prometheus/tsdb" - testutildb "github.com/prometheus/tsdb/testutil/db" ) func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { @@ -38,7 +37,7 @@ func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { safeDBOptions := *tsdb.DefaultOptions safeDBOptions.RetentionDuration = 0 - testutildb.CreateBlock(nil, tmpdir, testutildb.GenSeries(1, 1, 0, 1)) + tsdb.MockCreateBlock(nil, tmpdir, tsdb.MockGenSeries(1, 1, 0, 1)) dbRO, err := tsdb.OpenDBReadOnly(tmpdir, nil) if err != nil { @@ -206,15 +205,15 @@ func TestAnalyzeBlocks(t *testing.T) { fmt.Fprintf(&expected, "Postings (unique label pairs): %d\n", 1) fmt.Fprintf(&expected, "Postings entries (total label pairs): %d\n", 1) fmt.Fprintf(&expected, "\nLabel pairs most involved in churning:\n") - fmt.Fprintf(&expected, "1 %s=0", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "1 %s=0", tsdb.MockDefaultLabelName) fmt.Fprintf(&expected, "\nLabel names most involved in churning:\n") - fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "1 %s", tsdb.MockDefaultLabelName) fmt.Fprintf(&expected, "\nMost common label pairs:\n") - fmt.Fprintf(&expected, "1 %s=0", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "1 %s=0", tsdb.MockDefaultLabelName) fmt.Fprintf(&expected, "\nLabel names with highest cumulative label value length:\n") - fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "1 %s", tsdb.MockDefaultLabelName) fmt.Fprintf(&expected, "\nHighest cardinality labels:\n") - fmt.Fprintf(&expected, "1 %s", testutildb.DefaultLabelName) + fmt.Fprintf(&expected, "1 %s", tsdb.MockDefaultLabelName) fmt.Fprintf(&expected, "\nHighest cardinality metric names:\n") exp := expected.String() diff --git a/compact_test.go b/compact_test.go index 18990ed5..e19518eb 100644 --- a/compact_test.go +++ b/compact_test.go @@ -837,7 +837,7 @@ func BenchmarkCompaction(b *testing.B) { blockDirs := make([]string, 0, len(c.ranges)) var blocks []*Block for _, r := range c.ranges { - block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, r[0], r[1])), nil) + block, err := OpenBlock(nil, MockCreateBlock(b, dir, MockGenSeries(nSeries, 10, r[0], r[1])), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer func() { @@ -923,9 +923,9 @@ func TestCancelCompactions(t *testing.T) { }() // Create some blocks to fall within the compaction range. - createBlock(t, tmpdir, genSeries(10, 10000, 0, 1000)) - createBlock(t, tmpdir, genSeries(10, 10000, 1000, 2000)) - createBlock(t, tmpdir, genSeries(1, 1, 2000, 2001)) // The most recent block is ignored so can be e small one. + MockCreateBlock(t, tmpdir, MockGenSeries(10, 10000, 0, 1000)) + MockCreateBlock(t, tmpdir, MockGenSeries(10, 10000, 1000, 2000)) + MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 2000, 2001)) // The most recent block is ignored so can be e small one. // Copy the db so we have an exact copy to compare compaction times. tmpdirCopy := tmpdir + "Copy" @@ -1009,7 +1009,7 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) { {MinTime: 150, MaxTime: 200}, } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(1, 1, m.MinTime, m.MaxTime)) + MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) testutil.Equals(t, len(blocks), len(db.Blocks()), "unexpected block count after a reload") @@ -1032,7 +1032,7 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) { expBlocks := bootStrap(db) // Create a block that will trigger the reload to fail. - blockPath := createBlock(t, db.Dir(), genSeries(1, 1, 200, 300)) + blockPath := MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, 200, 300)) lastBlockIndex := path.Join(blockPath, indexFilename) actBlocks, err := blockDirs(db.Dir()) testutil.Ok(t, err) diff --git a/db_test.go b/db_test.go index 25fb8a7e..bf2335af 100644 --- a/db_test.go +++ b/db_test.go @@ -95,7 +95,7 @@ func TestDB_reloadOrder(t *testing.T) { {MinTime: 100, MaxTime: 110}, } for _, m := range metas { - createBlock(t, db.Dir(), genSeries(1, 1, m.MinTime, m.MaxTime)) + MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) @@ -986,7 +986,7 @@ func TestTombstoneCleanFail(t *testing.T) { // totalBlocks should be >=2 so we have enough blocks to trigger compaction failure. totalBlocks := 2 for i := 0; i < totalBlocks; i++ { - blockDir := createBlock(t, db.Dir(), genSeries(1, 1, 0, 1)) + blockDir := MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, 0, 1)) block, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) // Add some some fake tombstones to trigger the compaction. @@ -1030,7 +1030,7 @@ func (c *mockCompactorFailing) Write(dest string, b BlockReader, mint, maxt int6 return ulid.ULID{}, fmt.Errorf("the compactor already did the maximum allowed blocks so it is time to fail") } - block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, 0, 1)), nil) + block, err := OpenBlock(nil, MockCreateBlock(c.t, dest, MockGenSeries(1, 1, 0, 1)), nil) testutil.Ok(c.t, err) testutil.Ok(c.t, block.Close()) // Close block as we won't be using anywhere. c.blocks = append(c.blocks, block) @@ -1070,7 +1070,7 @@ func TestTimeRetention(t *testing.T) { } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(10, 10, m.MinTime, m.MaxTime)) + MockCreateBlock(t, db.Dir(), MockGenSeries(10, 10, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) // Reload the db to register the new blocks. @@ -1106,7 +1106,7 @@ func TestSizeRetention(t *testing.T) { } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime)) + MockCreateBlock(t, db.Dir(), MockGenSeries(100, 10, m.MinTime, m.MaxTime)) } // Test that registered size matches the actual disk size. @@ -1498,7 +1498,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - createBlock(t, dir, genSeries(1, 1, 1000, 2000)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, 1000, 2000)) db, err := Open(dir, nil, nil, nil) testutil.Ok(t, err) @@ -1514,7 +1514,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - createBlock(t, dir, genSeries(1, 1, 1000, 6000)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, 1000, 6000)) testutil.Ok(t, os.MkdirAll(path.Join(dir, "wal"), 0777)) w, err := wal.New(nil, nil, path.Join(dir, "wal"), false) @@ -1635,7 +1635,7 @@ func TestNoEmptyBlocks(t *testing.T) { {MinTime: currentTime + 100, MaxTime: currentTime + 100 + db.opts.BlockRanges[0]}, } for _, m := range blocks { - createBlock(t, db.Dir(), genSeries(2, 2, m.MinTime, m.MaxTime)) + MockCreateBlock(t, db.Dir(), MockGenSeries(2, 2, m.MinTime, m.MaxTime)) } oldBlocks := db.Blocks() @@ -2117,7 +2117,7 @@ func TestVerticalCompaction(t *testing.T) { }() for _, series := range c.blockSeries { - createBlock(t, tmpdir, series) + MockCreateBlock(t, tmpdir, series) } opts := *DefaultOptions opts.AllowOverlappingBlocks = true @@ -2177,7 +2177,7 @@ func TestBlockRanges(t *testing.T) { // Test that the compactor doesn't create overlapping blocks // when a non standard block already exists. firstBlockMaxT := int64(3) - createBlock(t, dir, genSeries(1, 1, 0, firstBlockMaxT)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, 0, firstBlockMaxT)) db, err := Open(dir, logger, nil, DefaultOptions) if err != nil { t.Fatalf("Opening test storage failed: %s", err) @@ -2227,7 +2227,7 @@ func TestBlockRanges(t *testing.T) { testutil.Ok(t, db.Close()) thirdBlockMaxt := secondBlockMaxt + 2 - createBlock(t, dir, genSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt)) db, err = Open(dir, logger, nil, DefaultOptions) if err != nil { @@ -2285,7 +2285,7 @@ func TestDBReadOnly(t *testing.T) { } for _, m := range dbBlocks { - createBlock(t, dbDir, genSeries(1, 1, m.MinTime, m.MaxTime)) + MockCreateBlock(t, dbDir, MockGenSeries(1, 1, m.MinTime, m.MaxTime)) } expSeriesCount++ } diff --git a/head_test.go b/head_test.go index 040ae828..7e7b349d 100644 --- a/head_test.go +++ b/head_test.go @@ -36,7 +36,7 @@ import ( ) func BenchmarkCreateSeries(b *testing.B) { - series := genSeries(b.N, 10, 0, 0) + series := MockGenSeries(b.N, 10, 0, 0) h, err := NewHead(nil, nil, nil, 10000) testutil.Ok(b, err) diff --git a/testutil/db/db.go b/mocks.go similarity index 56% rename from testutil/db/db.go rename to mocks.go index 11a5ad41..fa920620 100644 --- a/testutil/db/db.go +++ b/mocks.go @@ -1,17 +1,19 @@ -// Copyright 2019 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db +/* + * Copyright 2019 The Prometheus Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tsdb import ( "context" @@ -24,39 +26,29 @@ import ( "github.com/go-kit/kit/log" - "github.com/prometheus/tsdb" "github.com/prometheus/tsdb/labels" "github.com/prometheus/tsdb/testutil" "github.com/prometheus/tsdb/tsdbutil" ) +// This file holds types and functions that are used for testing +// purposes. + const ( - DefaultLabelName = "labelName" - defaultLabelValue = "labelValue" + MockDefaultLabelName = "labelName" + MockDefaultLabelValue = "labelValue" ) -type sample struct { - t int64 - v float64 -} - type mockSeries struct { labels func() labels.Labels - iterator func() tsdb.SeriesIterator -} - -func (s sample) T() int64 { - return s.t -} - -func (s sample) V() float64 { - return s.v + iterator func() SeriesIterator } -// CreateBlock creates a block with given set of series and returns its dir. -func CreateBlock(tb testing.TB, dir string, series []tsdb.Series) string { +// MockCreateBlock creates a block with given set of series and returns its dir. +// Intended for testing purposes. +func MockCreateBlock(tb testing.TB, dir string, series []Series) string { head := createHead(tb, series) - compactor, err := tsdb.NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) + compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) testutil.Ok(tb, err) testutil.Ok(tb, os.MkdirAll(dir, 0777)) @@ -68,8 +60,8 @@ func CreateBlock(tb testing.TB, dir string, series []tsdb.Series) string { return filepath.Join(dir, ulid.String()) } -func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { - head, err := tsdb.NewHead(nil, nil, nil, 2*60*60*1000) +func createHead(tb testing.TB, series []Series) *Head { + head, err := NewHead(nil, nil, nil, 2*60*60*1000) testutil.Ok(tb, err) defer head.Close() @@ -95,19 +87,20 @@ func createHead(tb testing.TB, series []tsdb.Series) *tsdb.Head { return head } -// GenSeries generates series with a given number of labels and values. -func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { +// MockGenSeries generates series with a given number of labels and values. +// Intended for testing purposes. +func MockGenSeries(totalSeries, labelCount int, mint, maxt int64) []Series { if totalSeries == 0 || labelCount == 0 { return nil } - series := make([]tsdb.Series, totalSeries) + series := make([]Series, totalSeries) for i := 0; i < totalSeries; i++ { lbls := make(map[string]string, labelCount) - lbls[DefaultLabelName] = strconv.Itoa(i) + lbls[MockDefaultLabelName] = strconv.Itoa(i) for j := 1; len(lbls) < labelCount; j++ { - lbls[DefaultLabelName+strconv.Itoa(j)] = defaultLabelValue + strconv.Itoa(j) + lbls[MockDefaultLabelName+strconv.Itoa(j)] = MockDefaultLabelValue + strconv.Itoa(j) } samples := make([]tsdbutil.Sample, 0, maxt-mint+1) for t := mint; t < maxt; t++ { @@ -118,14 +111,14 @@ func GenSeries(totalSeries, labelCount int, mint, maxt int64) []tsdb.Series { return series } -func newSeries(l map[string]string, s []tsdbutil.Sample) tsdb.Series { +func newSeries(l map[string]string, s []tsdbutil.Sample) Series { return &mockSeries{ labels: func() labels.Labels { return labels.FromMap(l) }, - iterator: func() tsdb.SeriesIterator { return newListSeriesIterator(s) }, + iterator: func() SeriesIterator { return newListSeriesIterator(s) }, } } -func (m *mockSeries) Labels() labels.Labels { return m.labels() } -func (m *mockSeries) Iterator() tsdb.SeriesIterator { return m.iterator() } +func (m *mockSeries) Labels() labels.Labels { return m.labels() } +func (m *mockSeries) Iterator() SeriesIterator { return m.iterator() } type listSeriesIterator struct { list []tsdbutil.Sample diff --git a/querier_test.go b/querier_test.go index 2be48fcd..069fc70c 100644 --- a/querier_test.go +++ b/querier_test.go @@ -1428,56 +1428,6 @@ func (m mockIndex) LabelNames() ([]string, error) { return labelNames, nil } -type mockSeries struct { - labels func() labels.Labels - iterator func() SeriesIterator -} - -func newSeries(l map[string]string, s []tsdbutil.Sample) Series { - return &mockSeries{ - labels: func() labels.Labels { return labels.FromMap(l) }, - iterator: func() SeriesIterator { return newListSeriesIterator(s) }, - } -} -func (m *mockSeries) Labels() labels.Labels { return m.labels() } -func (m *mockSeries) Iterator() SeriesIterator { return m.iterator() } - -type listSeriesIterator struct { - list []tsdbutil.Sample - idx int -} - -func newListSeriesIterator(list []tsdbutil.Sample) *listSeriesIterator { - return &listSeriesIterator{list: list, idx: -1} -} - -func (it *listSeriesIterator) At() (int64, float64) { - s := it.list[it.idx] - return s.T(), s.V() -} - -func (it *listSeriesIterator) Next() bool { - it.idx++ - return it.idx < len(it.list) -} - -func (it *listSeriesIterator) Seek(t int64) bool { - if it.idx == -1 { - it.idx = 0 - } - // Do binary search between current position and end. - it.idx = sort.Search(len(it.list)-it.idx, func(i int) bool { - s := it.list[i+it.idx] - return s.T() >= t - }) - - return it.idx < len(it.list) -} - -func (it *listSeriesIterator) Err() error { - return nil -} - func BenchmarkQueryIterator(b *testing.B) { cases := []struct { numBlocks int @@ -1516,14 +1466,14 @@ func BenchmarkQueryIterator(b *testing.B) { mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = genSeries(c.numSeries, 10, mint, maxt) + generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -1590,14 +1540,14 @@ func BenchmarkQuerySeek(b *testing.B) { mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = genSeries(c.numSeries, 10, mint, maxt) + generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -1735,14 +1685,14 @@ func BenchmarkSetMatcher(b *testing.B) { mint := i * int64(c.numSamplesPerSeriesPerBlock) maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = genSeries(c.numSeries, 10, mint, maxt) + generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, createBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -2090,8 +2040,8 @@ func TestClose(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - createBlock(t, dir, genSeries(1, 1, 0, 10)) - createBlock(t, dir, genSeries(1, 1, 10, 20)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, 0, 10)) + MockCreateBlock(t, dir, MockGenSeries(1, 1, 10, 20)) db, err := Open(dir, nil, nil, DefaultOptions) if err != nil { @@ -2155,7 +2105,7 @@ func BenchmarkQueries(b *testing.B) { testutil.Ok(b, os.RemoveAll(dir)) }() - series := genSeries(nSeries, 5, 1, int64(nSamples)) + series := MockGenSeries(nSeries, 5, 1, int64(nSamples)) // Add some common labels to make the matchers select these series. { @@ -2181,7 +2131,7 @@ func BenchmarkQueries(b *testing.B) { qs := []Querier{} for x := 0; x <= 10; x++ { - block, err := OpenBlock(nil, createBlock(b, dir, series), nil) + block, err := OpenBlock(nil, MockCreateBlock(b, dir, series), nil) testutil.Ok(b, err) q, err := NewBlockQuerier(block, 1, int64(nSamples)) testutil.Ok(b, err) From 5dad441d66a033be4bb98204aea8c0dd5a8fb731 Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 11 Aug 2019 13:30:03 +0200 Subject: [PATCH 12/13] Rename Mock{CreateBlock,GenSeries} to CreateBlock, GenSeries Signed-off-by: obitech --- block_test.go | 8 ++++---- cmd/tsdb/main_test.go | 2 +- compact_test.go | 12 ++++++------ db_test.go | 24 ++++++++++++------------ head_test.go | 2 +- mocks.go | 4 ++-- querier_test.go | 20 ++++++++++---------- 7 files changed, 36 insertions(+), 36 deletions(-) diff --git a/block_test.go b/block_test.go index 490ef149..cec33349 100644 --- a/block_test.go +++ b/block_test.go @@ -56,7 +56,7 @@ func TestSetCompactionFailed(t *testing.T) { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - blockDir := MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 1)) + blockDir := CreateBlock(t, tmpdir, GenSeries(1, 1, 0, 1)) b, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) testutil.Equals(t, false, b.meta.Compaction.Failed) @@ -76,7 +76,7 @@ func TestCreateBlock(t *testing.T) { defer func() { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - b, err := OpenBlock(nil, MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 10)), nil) + b, err := OpenBlock(nil, CreateBlock(t, tmpdir, GenSeries(1, 1, 0, 10)), nil) if err == nil { testutil.Ok(t, b.Close()) } @@ -133,7 +133,7 @@ func TestCorruptedChunk(t *testing.T) { testutil.Ok(t, os.RemoveAll(tmpdir)) }() - blockDir := MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 0, 1)) + blockDir := CreateBlock(t, tmpdir, GenSeries(1, 1, 0, 1)) files, err := sequenceFiles(chunkDir(blockDir)) testutil.Ok(t, err) testutil.Assert(t, len(files) > 0, "No chunk created.") @@ -167,7 +167,7 @@ func TestBlockSize(t *testing.T) { // Create a block and compare the reported size vs actual disk size. { - blockDirInit = MockCreateBlock(t, tmpdir, MockGenSeries(10, 1, 1, 100)) + blockDirInit = CreateBlock(t, tmpdir, GenSeries(10, 1, 1, 100)) blockInit, err = OpenBlock(nil, blockDirInit, nil) testutil.Ok(t, err) defer func() { diff --git a/cmd/tsdb/main_test.go b/cmd/tsdb/main_test.go index 543f664c..63d0429f 100644 --- a/cmd/tsdb/main_test.go +++ b/cmd/tsdb/main_test.go @@ -37,7 +37,7 @@ func createRoDb(t *testing.T) (*tsdb.DBReadOnly, func()) { safeDBOptions := *tsdb.DefaultOptions safeDBOptions.RetentionDuration = 0 - tsdb.MockCreateBlock(nil, tmpdir, tsdb.MockGenSeries(1, 1, 0, 1)) + tsdb.CreateBlock(nil, tmpdir, tsdb.GenSeries(1, 1, 0, 1)) dbRO, err := tsdb.OpenDBReadOnly(tmpdir, nil) if err != nil { diff --git a/compact_test.go b/compact_test.go index e19518eb..3824f856 100644 --- a/compact_test.go +++ b/compact_test.go @@ -837,7 +837,7 @@ func BenchmarkCompaction(b *testing.B) { blockDirs := make([]string, 0, len(c.ranges)) var blocks []*Block for _, r := range c.ranges { - block, err := OpenBlock(nil, MockCreateBlock(b, dir, MockGenSeries(nSeries, 10, r[0], r[1])), nil) + block, err := OpenBlock(nil, CreateBlock(b, dir, GenSeries(nSeries, 10, r[0], r[1])), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer func() { @@ -923,9 +923,9 @@ func TestCancelCompactions(t *testing.T) { }() // Create some blocks to fall within the compaction range. - MockCreateBlock(t, tmpdir, MockGenSeries(10, 10000, 0, 1000)) - MockCreateBlock(t, tmpdir, MockGenSeries(10, 10000, 1000, 2000)) - MockCreateBlock(t, tmpdir, MockGenSeries(1, 1, 2000, 2001)) // The most recent block is ignored so can be e small one. + CreateBlock(t, tmpdir, GenSeries(10, 10000, 0, 1000)) + CreateBlock(t, tmpdir, GenSeries(10, 10000, 1000, 2000)) + CreateBlock(t, tmpdir, GenSeries(1, 1, 2000, 2001)) // The most recent block is ignored so can be e small one. // Copy the db so we have an exact copy to compare compaction times. tmpdirCopy := tmpdir + "Copy" @@ -1009,7 +1009,7 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) { {MinTime: 150, MaxTime: 200}, } for _, m := range blocks { - MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, m.MinTime, m.MaxTime)) + CreateBlock(t, db.Dir(), GenSeries(1, 1, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) testutil.Equals(t, len(blocks), len(db.Blocks()), "unexpected block count after a reload") @@ -1032,7 +1032,7 @@ func TestDeleteCompactionBlockAfterFailedReload(t *testing.T) { expBlocks := bootStrap(db) // Create a block that will trigger the reload to fail. - blockPath := MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, 200, 300)) + blockPath := CreateBlock(t, db.Dir(), GenSeries(1, 1, 200, 300)) lastBlockIndex := path.Join(blockPath, indexFilename) actBlocks, err := blockDirs(db.Dir()) testutil.Ok(t, err) diff --git a/db_test.go b/db_test.go index bf2335af..3a37a5c3 100644 --- a/db_test.go +++ b/db_test.go @@ -95,7 +95,7 @@ func TestDB_reloadOrder(t *testing.T) { {MinTime: 100, MaxTime: 110}, } for _, m := range metas { - MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, m.MinTime, m.MaxTime)) + CreateBlock(t, db.Dir(), GenSeries(1, 1, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) @@ -986,7 +986,7 @@ func TestTombstoneCleanFail(t *testing.T) { // totalBlocks should be >=2 so we have enough blocks to trigger compaction failure. totalBlocks := 2 for i := 0; i < totalBlocks; i++ { - blockDir := MockCreateBlock(t, db.Dir(), MockGenSeries(1, 1, 0, 1)) + blockDir := CreateBlock(t, db.Dir(), GenSeries(1, 1, 0, 1)) block, err := OpenBlock(nil, blockDir, nil) testutil.Ok(t, err) // Add some some fake tombstones to trigger the compaction. @@ -1030,7 +1030,7 @@ func (c *mockCompactorFailing) Write(dest string, b BlockReader, mint, maxt int6 return ulid.ULID{}, fmt.Errorf("the compactor already did the maximum allowed blocks so it is time to fail") } - block, err := OpenBlock(nil, MockCreateBlock(c.t, dest, MockGenSeries(1, 1, 0, 1)), nil) + block, err := OpenBlock(nil, CreateBlock(c.t, dest, GenSeries(1, 1, 0, 1)), nil) testutil.Ok(c.t, err) testutil.Ok(c.t, block.Close()) // Close block as we won't be using anywhere. c.blocks = append(c.blocks, block) @@ -1070,7 +1070,7 @@ func TestTimeRetention(t *testing.T) { } for _, m := range blocks { - MockCreateBlock(t, db.Dir(), MockGenSeries(10, 10, m.MinTime, m.MaxTime)) + CreateBlock(t, db.Dir(), GenSeries(10, 10, m.MinTime, m.MaxTime)) } testutil.Ok(t, db.reload()) // Reload the db to register the new blocks. @@ -1106,7 +1106,7 @@ func TestSizeRetention(t *testing.T) { } for _, m := range blocks { - MockCreateBlock(t, db.Dir(), MockGenSeries(100, 10, m.MinTime, m.MaxTime)) + CreateBlock(t, db.Dir(), GenSeries(100, 10, m.MinTime, m.MaxTime)) } // Test that registered size matches the actual disk size. @@ -1498,7 +1498,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - MockCreateBlock(t, dir, MockGenSeries(1, 1, 1000, 2000)) + CreateBlock(t, dir, GenSeries(1, 1, 1000, 2000)) db, err := Open(dir, nil, nil, nil) testutil.Ok(t, err) @@ -1514,7 +1514,7 @@ func TestInitializeHeadTimestamp(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - MockCreateBlock(t, dir, MockGenSeries(1, 1, 1000, 6000)) + CreateBlock(t, dir, GenSeries(1, 1, 1000, 6000)) testutil.Ok(t, os.MkdirAll(path.Join(dir, "wal"), 0777)) w, err := wal.New(nil, nil, path.Join(dir, "wal"), false) @@ -1635,7 +1635,7 @@ func TestNoEmptyBlocks(t *testing.T) { {MinTime: currentTime + 100, MaxTime: currentTime + 100 + db.opts.BlockRanges[0]}, } for _, m := range blocks { - MockCreateBlock(t, db.Dir(), MockGenSeries(2, 2, m.MinTime, m.MaxTime)) + CreateBlock(t, db.Dir(), GenSeries(2, 2, m.MinTime, m.MaxTime)) } oldBlocks := db.Blocks() @@ -2117,7 +2117,7 @@ func TestVerticalCompaction(t *testing.T) { }() for _, series := range c.blockSeries { - MockCreateBlock(t, tmpdir, series) + CreateBlock(t, tmpdir, series) } opts := *DefaultOptions opts.AllowOverlappingBlocks = true @@ -2177,7 +2177,7 @@ func TestBlockRanges(t *testing.T) { // Test that the compactor doesn't create overlapping blocks // when a non standard block already exists. firstBlockMaxT := int64(3) - MockCreateBlock(t, dir, MockGenSeries(1, 1, 0, firstBlockMaxT)) + CreateBlock(t, dir, GenSeries(1, 1, 0, firstBlockMaxT)) db, err := Open(dir, logger, nil, DefaultOptions) if err != nil { t.Fatalf("Opening test storage failed: %s", err) @@ -2227,7 +2227,7 @@ func TestBlockRanges(t *testing.T) { testutil.Ok(t, db.Close()) thirdBlockMaxt := secondBlockMaxt + 2 - MockCreateBlock(t, dir, MockGenSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt)) + CreateBlock(t, dir, GenSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt)) db, err = Open(dir, logger, nil, DefaultOptions) if err != nil { @@ -2285,7 +2285,7 @@ func TestDBReadOnly(t *testing.T) { } for _, m := range dbBlocks { - MockCreateBlock(t, dbDir, MockGenSeries(1, 1, m.MinTime, m.MaxTime)) + CreateBlock(t, dbDir, GenSeries(1, 1, m.MinTime, m.MaxTime)) } expSeriesCount++ } diff --git a/head_test.go b/head_test.go index 7e7b349d..37d29378 100644 --- a/head_test.go +++ b/head_test.go @@ -36,7 +36,7 @@ import ( ) func BenchmarkCreateSeries(b *testing.B) { - series := MockGenSeries(b.N, 10, 0, 0) + series := GenSeries(b.N, 10, 0, 0) h, err := NewHead(nil, nil, nil, 10000) testutil.Ok(b, err) diff --git a/mocks.go b/mocks.go index fa920620..bafa1a9a 100644 --- a/mocks.go +++ b/mocks.go @@ -46,7 +46,7 @@ type mockSeries struct { // MockCreateBlock creates a block with given set of series and returns its dir. // Intended for testing purposes. -func MockCreateBlock(tb testing.TB, dir string, series []Series) string { +func CreateBlock(tb testing.TB, dir string, series []Series) string { head := createHead(tb, series) compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil) testutil.Ok(tb, err) @@ -89,7 +89,7 @@ func createHead(tb testing.TB, series []Series) *Head { // MockGenSeries generates series with a given number of labels and values. // Intended for testing purposes. -func MockGenSeries(totalSeries, labelCount int, mint, maxt int64) []Series { +func GenSeries(totalSeries, labelCount int, mint, maxt int64) []Series { if totalSeries == 0 || labelCount == 0 { return nil } diff --git a/querier_test.go b/querier_test.go index 069fc70c..5a4a4f8f 100644 --- a/querier_test.go +++ b/querier_test.go @@ -1466,14 +1466,14 @@ func BenchmarkQueryIterator(b *testing.B) { mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) + generatedSeries = GenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, CreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -1540,14 +1540,14 @@ func BenchmarkQuerySeek(b *testing.B) { mint := i*int64(c.numSamplesPerSeriesPerBlock) - offset maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) + generatedSeries = GenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, CreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -1685,14 +1685,14 @@ func BenchmarkSetMatcher(b *testing.B) { mint := i * int64(c.numSamplesPerSeriesPerBlock) maxt := mint + int64(c.numSamplesPerSeriesPerBlock) - 1 if len(prefilledLabels) == 0 { - generatedSeries = MockGenSeries(c.numSeries, 10, mint, maxt) + generatedSeries = GenSeries(c.numSeries, 10, mint, maxt) for _, s := range generatedSeries { prefilledLabels = append(prefilledLabels, s.Labels().Map()) } } else { generatedSeries = populateSeries(prefilledLabels, mint, maxt) } - block, err := OpenBlock(nil, MockCreateBlock(b, dir, generatedSeries), nil) + block, err := OpenBlock(nil, CreateBlock(b, dir, generatedSeries), nil) testutil.Ok(b, err) blocks = append(blocks, block) defer block.Close() @@ -2040,8 +2040,8 @@ func TestClose(t *testing.T) { testutil.Ok(t, os.RemoveAll(dir)) }() - MockCreateBlock(t, dir, MockGenSeries(1, 1, 0, 10)) - MockCreateBlock(t, dir, MockGenSeries(1, 1, 10, 20)) + CreateBlock(t, dir, GenSeries(1, 1, 0, 10)) + CreateBlock(t, dir, GenSeries(1, 1, 10, 20)) db, err := Open(dir, nil, nil, DefaultOptions) if err != nil { @@ -2105,7 +2105,7 @@ func BenchmarkQueries(b *testing.B) { testutil.Ok(b, os.RemoveAll(dir)) }() - series := MockGenSeries(nSeries, 5, 1, int64(nSamples)) + series := GenSeries(nSeries, 5, 1, int64(nSamples)) // Add some common labels to make the matchers select these series. { @@ -2131,7 +2131,7 @@ func BenchmarkQueries(b *testing.B) { qs := []Querier{} for x := 0; x <= 10; x++ { - block, err := OpenBlock(nil, MockCreateBlock(b, dir, series), nil) + block, err := OpenBlock(nil, CreateBlock(b, dir, series), nil) testutil.Ok(b, err) q, err := NewBlockQuerier(block, 1, int64(nSamples)) testutil.Ok(b, err) From c6f75f5884b8328e2ec348d70f1dc73af206e196 Mon Sep 17 00:00:00 2001 From: obitech Date: Sun, 11 Aug 2019 15:04:52 +0200 Subject: [PATCH 13/13] Fix comments for CreateBlock and GenSeries Signed-off-by: obitech --- mocks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mocks.go b/mocks.go index bafa1a9a..d11103ae 100644 --- a/mocks.go +++ b/mocks.go @@ -44,7 +44,7 @@ type mockSeries struct { iterator func() SeriesIterator } -// MockCreateBlock creates a block with given set of series and returns its dir. +// CreateBlock creates a block with given set of series and returns its dir. // Intended for testing purposes. func CreateBlock(tb testing.TB, dir string, series []Series) string { head := createHead(tb, series) @@ -87,7 +87,7 @@ func createHead(tb testing.TB, series []Series) *Head { return head } -// MockGenSeries generates series with a given number of labels and values. +// GenSeries generates series with a given number of labels and values. // Intended for testing purposes. func GenSeries(totalSeries, labelCount int, mint, maxt int64) []Series { if totalSeries == 0 || labelCount == 0 {