test: Support testing cgroups v1 and v2

- We added two test cases for both v1 and v2

- sys.ttar file contains control groups for two cases

- We check cgroups mode in e2e-test.sh and compare output with correct file

Makefile

@@ -12,8 +12,10 @@ PROMTOOL_VERSION ?= 2.30.0
 PROMTOOL_URL     ?= https://github.com/prometheus/prometheus/releases/download/v$(PROMTOOL_VERSION)/prometheus-$(PROMTOOL_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 PROMTOOL         ?= $(FIRST_GOPATH)/bin/promtool
 
+TEST_DOCKER             ?= false
 DOCKER_IMAGE_NAME       ?= batchjob-exporter
 MACH                    ?= $(shell uname -m)
+CGROUPS_MODE            ?= $([ $(stat -fc %T /sys/fs/cgroup/) = "cgroup2fs" ] && echo "unified" || ( [ -e /sys/fs/cgroup/unified/ ] && echo "hybrid" || echo "legacy"))
 
 STATICCHECK_IGNORE =
 
@@ -23,6 +25,12 @@ else
 	test-e2e := skip-test-e2e
 endif
 
+ifeq ($(TEST_DOCKER), false)
+	test-docker := skip-test-docker
+else
+	test-docker := test-docker
+endif
+
 # Use CGO for non-Linux builds.
 ifeq ($(GOOS), linux)
 	PROMU_CONF ?= .promu.yml
@@ -49,7 +57,14 @@ endif
 
 PROMU := $(FIRST_GOPATH)/bin/promu --config $(PROMU_CONF)
 
-e2e-out = collector/fixtures/e2e-test-output.txt
+e2e-cgroupsv2-out = collector/fixtures/e2e-test-cgroupsv2-output.txt
+e2e-cgroupsv1-out = collector/fixtures/e2e-test-cgroupsv1-output.txt
+
+ifeq ($(CGROUPS_MODE), unified)
+	e2e-out = $(e2e-cgroupsv2-out)
+else
+	e2e-out = $(e2e-cgroupsv1-out)
+endif
 
 # 64bit -> 32bit mapping for cross-checking. At least for amd64/386, the 64bit CPU can execute 32bit code but not the other way around, so we don't support cross-testing upwards.
 cross-test = skip-test-32bit
@@ -67,7 +82,7 @@ $(eval $(call goarch_pair,amd64,386))
 $(eval $(call goarch_pair,mips64,mips))
 $(eval $(call goarch_pair,mips64el,mipsel))
 
-all:: vet checkmetrics checkrules common-all $(cross-test) $(test-e2e)
+all:: vet checkmetrics checkrules common-all $(cross-test) $(test-docker) $(test-e2e)
 
 .PHONY: test
 test: collector/fixtures/sys/.unpacked 
@@ -93,7 +108,6 @@ update_fixtures:
 	rm -vf collector/fixtures/sys/.unpacked
 	./ttar -C collector/fixtures -c -f collector/fixtures/sys.ttar sys
 
-
 .PHONY: test-e2e
 test-e2e: build collector/fixtures/sys/.unpacked 
 	@echo ">> running end-to-end tests"
@@ -113,10 +127,14 @@ checkrules: $(PROMTOOL)
 	@echo ">> checking rules for correctness"
 	find . -name "*rules*.yml" | xargs -I {} $(PROMTOOL) check rules {}
 
-# .PHONY: test-docker
-# test-docker:
-# 	@echo ">> testing docker image"
-# 	./test_image.sh "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-amd64:$(DOCKER_IMAGE_TAG)" 9100
+.PHONY: test-docker
+test-docker:
+	@echo ">> testing docker image"
+	./test_image.sh "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-amd64:$(DOCKER_IMAGE_TAG)" 9010
+
+.PHONY: skip-test-docker
+skip-test-docker:
+	@echo ">> SKIP running docker tests"
 
 .PHONY: promtool
 promtool: $(PROMTOOL)

collector/fixtures/e2e-test-cgroupsv1-output.txt

@@ -0,0 +1,173 @@
+# HELP batchjob_cpu_system_seconds Cumulative CPU system seconds for jobid
+# TYPE batchjob_cpu_system_seconds gauge
+batchjob_cpu_system_seconds{batch="slurm",jobid="1009248",step="",task=""} 0.45
+# HELP batchjob_cpu_total_seconds Cumulative CPU total seconds for jobid
+# TYPE batchjob_cpu_total_seconds gauge
+batchjob_cpu_total_seconds{batch="slurm",jobid="1009248",step="",task=""} 1.012410966
+# HELP batchjob_cpu_user_seconds Cumulative CPU user seconds for jobid
+# TYPE batchjob_cpu_user_seconds gauge
+batchjob_cpu_user_seconds{batch="slurm",jobid="1009248",step="",task=""} 0.39
+# HELP batchjob_cpus Number of CPUs in the jobid
+# TYPE batchjob_cpus gauge
+batchjob_cpus{batch="slurm",jobid="1009248",step="",task=""} 0
+# HELP batchjob_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which batchjob_exporter was built, and the goos and goarch for the build.
+# TYPE batchjob_exporter_build_info gauge
+batchjob_exporter_build_info{branch="",goarch="amd64",goos="linux",goversion="go1.21.4",revision="unknown",tags="unknown",version=""} 1
+# HELP batchjob_memory_cache_bytes Memory cache used in bytes
+# TYPE batchjob_memory_cache_bytes gauge
+batchjob_memory_cache_bytes{batch="slurm",jobid="1009248",step="",task=""} 2.1086208e+07
+# HELP batchjob_memory_fail_count Memory fail count
+# TYPE batchjob_memory_fail_count gauge
+batchjob_memory_fail_count{batch="slurm",jobid="1009248",step="",task=""} 0
+# HELP batchjob_memory_rss_bytes Memory RSS used in bytes
+# TYPE batchjob_memory_rss_bytes gauge
+batchjob_memory_rss_bytes{batch="slurm",jobid="1009248",step="",task=""} 1.0407936e+07
+# HELP batchjob_memory_total_bytes Memory total given to jobid in bytes
+# TYPE batchjob_memory_total_bytes gauge
+batchjob_memory_total_bytes{batch="slurm",jobid="1009248",step="",task=""} 2.01362030592e+11
+# HELP batchjob_memory_used_bytes Memory used in bytes
+# TYPE batchjob_memory_used_bytes gauge
+batchjob_memory_used_bytes{batch="slurm",jobid="1009248",step="",task=""} 4.0194048e+07
+# HELP batchjob_memsw_fail_count Swap fail count
+# TYPE batchjob_memsw_fail_count gauge
+batchjob_memsw_fail_count{batch="slurm",jobid="1009248",step="",task=""} 0
+# HELP batchjob_memsw_total_bytes Swap total given to jobid in bytes
+# TYPE batchjob_memsw_total_bytes gauge
+batchjob_memsw_total_bytes{batch="slurm",jobid="1009248",step="",task=""} 9.223372036854772e+18
+# HELP batchjob_memsw_used_bytes Swap used in bytes
+# TYPE batchjob_memsw_used_bytes gauge
+batchjob_memsw_used_bytes{batch="slurm",jobid="1009248",step="",task=""} 4.032512e+07
+# HELP batchjob_rapl_package_joules_total Current RAPL package value in joules
+# TYPE batchjob_rapl_package_joules_total counter
+batchjob_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 258218.293244
+batchjob_rapl_package_joules_total{index="1",path="collector/fixtures/sys/class/powercap/intel-rapl:1"} 130570.505826
+# HELP batchjob_scrape_collector_duration_seconds batchjob_exporter: Duration of a collector scrape.
+# TYPE batchjob_scrape_collector_duration_seconds gauge
+batchjob_scrape_collector_duration_seconds{collector="ipmi_dcmi"} 0.000502612
+batchjob_scrape_collector_duration_seconds{collector="rapl"} 0.002879363
+batchjob_scrape_collector_duration_seconds{collector="slurm_job"} 0.003768202
+# HELP batchjob_scrape_collector_success batchjob_exporter: Whether a collector succeeded.
+# TYPE batchjob_scrape_collector_success gauge
+batchjob_scrape_collector_success{collector="ipmi_dcmi"} 0
+batchjob_scrape_collector_success{collector="rapl"} 1
+batchjob_scrape_collector_success{collector="slurm_job"} 1
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 0
+go_gc_duration_seconds{quantile="0.25"} 0
+go_gc_duration_seconds{quantile="0.5"} 0
+go_gc_duration_seconds{quantile="0.75"} 0
+go_gc_duration_seconds{quantile="1"} 0
+go_gc_duration_seconds_sum 0
+go_gc_duration_seconds_count 0
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 7
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.21.4"} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 681656
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
+# TYPE go_memstats_alloc_bytes_total counter
+go_memstats_alloc_bytes_total 681656
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+go_memstats_buck_hash_sys_bytes 1.445185e+06
+# HELP go_memstats_frees_total Total number of frees.
+# TYPE go_memstats_frees_total counter
+go_memstats_frees_total 246
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
+# TYPE go_memstats_gc_sys_bytes gauge
+go_memstats_gc_sys_bytes 2.915544e+06
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
+# TYPE go_memstats_heap_alloc_bytes gauge
+go_memstats_heap_alloc_bytes 681656
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
+# TYPE go_memstats_heap_idle_bytes gauge
+go_memstats_heap_idle_bytes 1.982464e+06
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
+# TYPE go_memstats_heap_inuse_bytes gauge
+go_memstats_heap_inuse_bytes 1.851392e+06
+# HELP go_memstats_heap_objects Number of allocated objects.
+# TYPE go_memstats_heap_objects gauge
+go_memstats_heap_objects 3643
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
+# TYPE go_memstats_heap_released_bytes gauge
+go_memstats_heap_released_bytes 1.949696e+06
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
+# TYPE go_memstats_heap_sys_bytes gauge
+go_memstats_heap_sys_bytes 3.833856e+06
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+go_memstats_last_gc_time_seconds 0
+# HELP go_memstats_lookups_total Total number of pointer lookups.
+# TYPE go_memstats_lookups_total counter
+go_memstats_lookups_total 0
+# HELP go_memstats_mallocs_total Total number of mallocs.
+# TYPE go_memstats_mallocs_total counter
+go_memstats_mallocs_total 3889
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+go_memstats_mcache_inuse_bytes 1200
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
+# TYPE go_memstats_mcache_sys_bytes gauge
+go_memstats_mcache_sys_bytes 15600
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+go_memstats_mspan_inuse_bytes 33768
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
+# TYPE go_memstats_mspan_sys_bytes gauge
+go_memstats_mspan_sys_bytes 65184
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
+# TYPE go_memstats_next_gc_bytes gauge
+go_memstats_next_gc_bytes 4.194304e+06
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
+# TYPE go_memstats_other_sys_bytes gauge
+go_memstats_other_sys_bytes 511839
+# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
+# TYPE go_memstats_stack_inuse_bytes gauge
+go_memstats_stack_inuse_bytes 360448
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
+# TYPE go_memstats_stack_sys_bytes gauge
+go_memstats_stack_sys_bytes 360448
+# HELP go_memstats_sys_bytes Number of bytes obtained from system.
+# TYPE go_memstats_sys_bytes gauge
+go_memstats_sys_bytes 9.147656e+06
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+go_threads 7
+# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
+# TYPE process_cpu_seconds_total counter
+process_cpu_seconds_total 0
+# HELP process_max_fds Maximum number of open file descriptors.
+# TYPE process_max_fds gauge
+process_max_fds 1.048576e+06
+# HELP process_open_fds Number of open file descriptors.
+# TYPE process_open_fds gauge
+process_open_fds 8
+# HELP process_resident_memory_bytes Resident memory size in bytes.
+# TYPE process_resident_memory_bytes gauge
+process_resident_memory_bytes 1.376256e+07
+# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
+# TYPE process_start_time_seconds gauge
+process_start_time_seconds 1.70006874948e+09
+# HELP process_virtual_memory_bytes Virtual memory size in bytes.
+# TYPE process_virtual_memory_bytes gauge
+process_virtual_memory_bytes 1.647849472e+09
+# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
+# TYPE process_virtual_memory_max_bytes gauge
+process_virtual_memory_max_bytes 1.8446744073709552e+19
+# HELP promhttp_metric_handler_errors_total Total number of internal errors encountered by the promhttp metric handler.
+# TYPE promhttp_metric_handler_errors_total counter
+promhttp_metric_handler_errors_total{cause="encoding"} 0
+promhttp_metric_handler_errors_total{cause="gathering"} 0
+# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
+# TYPE promhttp_metric_handler_requests_in_flight gauge
+promhttp_metric_handler_requests_in_flight 1
+# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
+# TYPE promhttp_metric_handler_requests_total counter
+promhttp_metric_handler_requests_total{code="200"} 0
+promhttp_metric_handler_requests_total{code="500"} 0
+promhttp_metric_handler_requests_total{code="503"} 0