Skip to content

Commit

Permalink
test(e2e): e2e test for reverting the instrumentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@basti1302
basti1302 committed May 29, 2024
1 parent 50ff86b commit ab88e01
Show file tree
Hide file tree
Showing 2 changed files with 230 additions and 94 deletions.
191 changes: 134 additions & 57 deletions test/e2e/e2e_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func RenderTemplates() {
ExpectWithOffset(1, RunAndIgnoreOutput(exec.Command("test-resources/bin/render-templates.sh"))).To(Succeed())
}

func EnsureNamespaceExists(namespace string) bool {
func RecreateNamespace(namespace string) bool {
err := RunAndIgnoreOutput(exec.Command("kubectl", "get", "ns", namespace), false)
if err != nil {
By(fmt.Sprintf("creating namespace %s", namespace))
Expand Down Expand Up @@ -358,7 +358,7 @@ func DeployDash0Resource(namespace string) {
func UndeployDash0Resource(namespace string) {
ExpectWithOffset(1,
RunAndIgnoreOutput(exec.Command(
"kubectl", "delete", "-n", namespace, "-k", "config/samples"))).To(Succeed())
"kubectl", "delete", "-n", namespace, "-k", "config/samples", "--ignore-not-found"))).To(Succeed())
}

func InstallNodeJsCronJob(namespace string) error {
Expand Down Expand Up @@ -525,51 +525,123 @@ func DeleteTestIdFiles() {
_ = os.Remove("test-resources/e2e-test-volumes/test-uuid/job.test.id")
}

func VerifyThatSpansAreCaptured(
func VerifyThatWorkloadHasBeenInstrumented(
namespace string,
kind string,
sendRequests bool,
workloadType string,
isBatch bool,
restartPodsManually bool,
instrumentationBy string,
) {
By("verify that the workload has been instrumented and is sending telemetry")
) string {
By("waiting for the workload to get instrumented by polling its labels")
Eventually(func(g Gomega) {
verifyLabels(g, namespace, workloadType, true, instrumentationBy)
}, verifyTelemetryTimeout, verifyTelemetryPollingInterval).Should(Succeed())

if restartPodsManually {
restartAllPods(namespace)
}

By("waiting for spans to be captured")
var testId string
if sendRequests {
if isBatch {
By(fmt.Sprintf("waiting for the test ID file to be written by the %s under test", workloadType))
Eventually(func(g Gomega) {
// For resource types like batch jobs/cron jobs, the application under test generates the test ID and writes it
// to a volume that maps to a host path. We read the test ID from the host path and use it to verify the spans.
testIdBytes, err := os.ReadFile(fmt.Sprintf("test-resources/e2e-test-volumes/test-uuid/%s.test.id", workloadType))
g.Expect(err).NotTo(HaveOccurred())
testId = string(testIdBytes)
// Also, cronjob pods are only scheduled once a minute, so we might need to wait a while for the ID to
// become available, hence the 80 second timeout for the surrounding Eventually.
}, 80*time.Second, 200*time.Millisecond).Should(Succeed())
} else {
// For resource types that are available as a service (daemonset, deployment etc.) we send HTTP requests with
// a unique ID as a query parameter. When checking the produced spans that the OTel collector writes to disk via
// the file exporter, we can verify that the span is actually from the currently running test case by inspecting
// the http.target span attribute. This guarantees that we do not accidentally pass the test due to a span from
// a previous test case.
testIdUuid := uuid.New()
testId = testIdUuid.String()
} else {
By(fmt.Sprintf("waiting for the test ID file to be written by the %s under test", kind))
Eventually(func(g Gomega) {
// For resource types like batch jobs/cron jobs, the application under test generates the test ID and writes it
// to a volume that maps to a host path. We read the test ID from the host path and use it to verify the spans.
testIdBytes, err := os.ReadFile(fmt.Sprintf("test-resources/e2e-test-volumes/test-uuid/%s.test.id", kind))
g.Expect(err).NotTo(HaveOccurred())
testId = string(testIdBytes)
}, 80*time.Second, 200*time.Millisecond).Should(Succeed())
}

httpPathWithQuery := fmt.Sprintf("/dash0-k8s-operator-test?id=%s", testId)
Eventually(func(g Gomega) {
verifySpans(g, isBatch, httpPathWithQuery)
}, verifyTelemetryTimeout, verifyTelemetryPollingInterval).Should(Succeed())
By("matchin spans have been received")
return testId
}

By("waiting for the workload to be modified/checking labels")
func VerifyThatInstrumentationHasBeenReverted(
namespace string,
workloadType string,
isBatch bool,
restartPodsManually bool,
testId string,
) {
By("waiting for the instrumentation to get removed from the workload by polling its labels")
Eventually(func(g Gomega) {
verifyLabels(g, namespace, kind, true, instrumentationBy)
verifyLabelsHaveBeenRemoved(g, namespace, workloadType)
}, verifyTelemetryTimeout, verifyTelemetryPollingInterval).Should(Succeed())

if restartPodsManually {
restartAllPods(namespace)
}

By("waiting for spans to be captured")
Eventually(func(g Gomega) {
verifySpans(g, sendRequests, httpPathWithQuery)
}, verifyTelemetryTimeout, verifyTelemetryPollingInterval).Should(Succeed())
By("matchin spans have been received")
// Add some buffer time between the workloads being restarted and verifying that no spans are produced/captured.
time.Sleep(10 * time.Second)

By("verifying that spans are no longer being captured")
httpPathWithQuery := fmt.Sprintf("/dash0-k8s-operator-test?id=%s", testId)
timeToCheckForSpans := 20 * time.Second
if workloadType == "cronjob" {
// Pod for cron jobs only get scheduled once a minute, since the cronjob schedule format does not allow for jobs
// starting every second. Thus, to make the test valid, we need to monitor for spans a little bit longer than
// for appsv1 workloads.
timeToCheckForSpans = 80 * time.Second
}
Consistently(func(g Gomega) {
verifyNoSpans(isBatch, httpPathWithQuery)
}, timeToCheckForSpans, 1*time.Second).Should(Succeed())

By("matching spans are no longer captured")
}

func verifyLabels(g Gomega, namespace string, kind string, hasBeenInstrumented bool, instrumentationBy string) {
instrumented := readLabel(g, namespace, kind, "dash0.instrumented")
g.ExpectWithOffset(1, instrumented).To(Equal(strconv.FormatBool(hasBeenInstrumented)))
operatorVersion := readLabel(g, namespace, kind, "dash0.operator.version")
g.ExpectWithOffset(1, operatorVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+"))
initContainerImageVersion := readLabel(g, namespace, kind, "dash0.initcontainer.image.version")
g.ExpectWithOffset(1, initContainerImageVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+"))
instrumentedBy := readLabel(g, namespace, kind, "dash0.instrumented.by")
g.ExpectWithOffset(1, instrumentedBy).To(Equal(instrumentationBy))
}

func verifyLabelsHaveBeenRemoved(g Gomega, namespace string, kind string) {
instrumented := readLabel(g, namespace, kind, "dash0.instrumented")
g.ExpectWithOffset(1, instrumented).To(Equal(""))
operatorVersion := readLabel(g, namespace, kind, "dash0.operator.version")
g.ExpectWithOffset(1, operatorVersion).To(Equal(""))
initContainerImageVersion := readLabel(g, namespace, kind, "dash0.initcontainer.image.version")
g.ExpectWithOffset(1, initContainerImageVersion).To(Equal(""))
instrumentedBy := readLabel(g, namespace, kind, "dash0.instrumented.by")
g.ExpectWithOffset(1, instrumentedBy).To(Equal(""))
}

func readLabel(g Gomega, namespace string, kind string, labelKey string) string {
labelValue, err := Run(exec.Command(
"kubectl",
"get",
kind,
"--namespace",
namespace,
fmt.Sprintf("dash0-operator-nodejs-20-express-test-%s", kind),
"-o",
fmt.Sprintf("jsonpath={.metadata.labels['%s']}", strings.ReplaceAll(labelKey, ".", "\\.")),
), false)
g.ExpectWithOffset(1, err).NotTo(HaveOccurred())
return string(labelValue)
}

func restartAllPods(namespace string) {
Expand All @@ -591,13 +663,40 @@ func restartAllPods(namespace string) {

}

func verifySpans(g Gomega, sendRequests bool, httpPathWithQuery string) {
if sendRequests {
func verifySpans(g Gomega, isBatch bool, httpPathWithQuery string) {
spansFound := sendRequestAndFindMatchingSpans(g, isBatch, httpPathWithQuery, true, nil)
g.Expect(spansFound).To(BeTrue(), "expected to find at least one matching HTTP server span")
}

func verifyNoSpans(isBatch bool, httpPathWithQuery string) {
timestampLowerBound := time.Now()
spansFound := sendRequestAndFindMatchingSpans(Default, isBatch, httpPathWithQuery, false, &timestampLowerBound)
Expect(spansFound).To(BeFalse(), "expected to find no matching HTTP server span")
}

func sendRequestAndFindMatchingSpans(
g Gomega,
isBatch bool,
httpPathWithQuery string,
requestsMustNotFail bool,
timestampLowerBound *time.Time,
) bool {
sendRequest(g, isBatch, httpPathWithQuery, requestsMustNotFail)
return fileHasMatchingSpan(g, httpPathWithQuery, timestampLowerBound)
}

func sendRequest(g Gomega, isBatch bool, httpPathWithQuery string, mustNotFail bool) {
if !isBatch {
response, err := Run(exec.Command("curl", fmt.Sprintf("http://localhost:1207%s", httpPathWithQuery)), false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(string(response)).To(ContainSubstring(
"We make Observability easy for every developer."))
if mustNotFail {
g.Expect(err).NotTo(HaveOccurred())
g.Expect(string(response)).To(ContainSubstring(
"We make Observability easy for every developer."))
}
}
}

func fileHasMatchingSpan(g Gomega, httpPathWithQuery string, timestampLowerBound *time.Time) bool {
fileHandle, err := os.Open("test-resources/e2e-test-volumes/collector-received-data/traces.jsonl")
g.Expect(err).NotTo(HaveOccurred())
defer func() {
Expand All @@ -618,48 +717,26 @@ func verifySpans(g Gomega, sendRequests bool, httpPathWithQuery string) {
if spansFound = hasMatchingSpans(
traces,
isHttpServerSpanWithHttpTarget(httpPathWithQuery),
timestampLowerBound,
); spansFound {
break
}
}
g.Expect(scanner.Err()).NotTo(HaveOccurred())
g.Expect(spansFound).To(BeTrue(), "expected to find an HTTP server span")
}

func verifyLabels(g Gomega, namespace string, kind string, hasBeenInstrumented bool, instrumentationBy string) {
instrumented := readLabel(g, namespace, kind, "dash0.instrumented")
g.ExpectWithOffset(1, instrumented).To(Equal(strconv.FormatBool(hasBeenInstrumented)))
operatorVersion := readLabel(g, namespace, kind, "dash0.operator.version")
g.ExpectWithOffset(1, operatorVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+"))
initContainerImageVersion := readLabel(g, namespace, kind, "dash0.initcontainer.image.version")
g.ExpectWithOffset(1, initContainerImageVersion).To(MatchRegexp("\\d+\\.\\d+\\.\\d+"))
instrumentedBy := readLabel(g, namespace, kind, "dash0.instrumented.by")
g.ExpectWithOffset(1, instrumentedBy).To(Equal(instrumentationBy))
}
g.Expect(scanner.Err()).NotTo(HaveOccurred())

func readLabel(g Gomega, namespace string, kind string, labelKey string) string {
labelValue, err := Run(exec.Command(
"kubectl",
"get",
kind,
"--namespace",
namespace,
fmt.Sprintf("dash0-operator-nodejs-20-express-test-%s", kind),
"-o",
fmt.Sprintf("jsonpath={.metadata.labels['%s']}", strings.ReplaceAll(labelKey, ".", "\\.")),
), false)
g.ExpectWithOffset(1, err).NotTo(HaveOccurred())
return string(labelValue)
return spansFound
}

func hasMatchingSpans(traces ptrace.Traces, matchFn func(span ptrace.Span) bool) bool {
func hasMatchingSpans(traces ptrace.Traces, matchFn func(span ptrace.Span) bool, timestampLowerBound *time.Time) bool {
for i := 0; i < traces.ResourceSpans().Len(); i++ {
resourceSpan := traces.ResourceSpans().At(i)
for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ {
scopeSpan := resourceSpan.ScopeSpans().At(j)
for k := 0; k < scopeSpan.Spans().Len(); k++ {
span := scopeSpan.Spans().At(k)
if matchFn(span) {
if (timestampLowerBound == nil || span.StartTimestamp().AsTime().After(*timestampLowerBound)) &&
matchFn(span) {
return true
}
}
Expand Down
Loading

0 comments on commit ab88e01

Please sign in to comment.