Use pkg structure

.gitignore

@@ -21,7 +21,7 @@ vendor/
 go.work
 
 # Ignore fixtures
-collector/fixtures/sys
+pkg/collector/fixtures/sys
 
 # Ignore scripts
 run.sh
@@ -34,3 +34,8 @@ run.sh
 
 # Ignore binaries
 /bin
+
+# Ignore test files
+*.db
+*.prof
+lasttimestamp

.promu-cgo.yml

@@ -4,7 +4,7 @@ go:
     version: 1.21
     cgo: true
 repository:
-    path: github.com/mahendrapaipuri/batchjob_exporter
+    path: github.com/mahendrapaipuri/batchjob_monitoring
 build:
     binaries:
         - name: batchjob_exporter

.promu.yml

@@ -3,7 +3,7 @@ go:
     # .promu-cgo.yml should also be updated.
     version: 1.21
 repository:
-    path: github.com/mahendrapaipuri/batchjob_exporter
+    path: github.com/mahendrapaipuri/batchjob_monitoring
 build:
     binaries:
         - name: batchjob_exporter

Makefile

@@ -59,8 +59,8 @@ endif
 
 PROMU := $(FIRST_GOPATH)/bin/promu --config $(PROMU_CONF)
 
-e2e-cgroupsv2-out = collector/fixtures/e2e-test-cgroupsv2-output.txt
-e2e-cgroupsv1-out = collector/fixtures/e2e-test-cgroupsv1-output.txt
+e2e-cgroupsv2-out = pkg/collector/fixtures/e2e-test-cgroupsv2-output.txt
+e2e-cgroupsv1-out = pkg/collector/fixtures/e2e-test-cgroupsv1-output.txt
 
 ifeq ($(CGROUPS_MODE), unified)
 	e2e-out = $(e2e-cgroupsv2-out)
@@ -87,12 +87,12 @@ $(eval $(call goarch_pair,mips64el,mipsel))
 all:: vet checkmetrics checkrules common-all $(cross-test) $(test-docker) $(test-e2e)
 
 .PHONY: test
-test: collector/fixtures/sys/.unpacked 
+test: pkg/collector/fixtures/sys/.unpacked 
 	@echo ">> running tests"
 	$(GO) test -short $(test-flags) $(pkgs)
 
 .PHONY: test-32bit
-test-32bit: collector/fixtures/sys/.unpacked 
+test-32bit: pkg/collector/fixtures/sys/.unpacked 
 	@echo ">> running tests in 32-bit mode"
 	@env GOARCH=$(GOARCH_CROSS) $(GO) test $(pkgs)
 
@@ -107,11 +107,11 @@ skip-test-32bit:
 	touch $@
 
 update_fixtures:
-	rm -vf collector/fixtures/sys/.unpacked
-	./scripts/ttar -C collector/fixtures -c -f collector/fixtures/sys.ttar sys
+	rm -vf pkg/collector/fixtures/sys/.unpacked
+	./scripts/ttar -C pkg/collector/fixtures -c -f pkg/collector/fixtures/sys.ttar sys
 
 .PHONY: test-e2e
-test-e2e: build collector/fixtures/sys/.unpacked 
+test-e2e: build pkg/collector/fixtures/sys/.unpacked 
 	@echo ">> running end-to-end tests"
 	./scripts/e2e-test.sh
 

cmd/batchjob_exporter/batchjob_exporter.go

@@ -18,7 +18,7 @@ import (
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/mahendrapaipuri/batchjob_exporter/collector"
+	"github.com/mahendrapaipuri/batchjob_monitoring/pkg/collector"
 	"github.com/prometheus/client_golang/prometheus"
 	promcollectors "github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"

go.mod

@@ -1,4 +1,4 @@
-module github.com/mahendrapaipuri/batchjob_exporter
+module github.com/mahendrapaipuri/batchjob_monitoring
 
 go 1.21
 
@@ -24,7 +24,7 @@ require (
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect

go.sum

@@ -34,8 +34,8 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
-github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
+github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -81,6 +81,7 @@ golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

collector/emissions.go → pkg/collector/emissions.go

@@ -13,6 +13,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+
+	utils "github.com/mahendrapaipuri/batchjob_monitoring/pkg/utils"
 )
 
 const emissionsCollectorSubsystem = "emissions"
@@ -30,7 +32,7 @@ var (
 	countryCode            = kingpin.Flag("collector.emissions.country.code", "ISO 3166-1 alpha-3 Country code. OWID energy data [https://github.com/owid/energy-data] estimated constant emission factor is used for all countries except for France. A real time emission factor will be used for France from RTE eCO2 mix [https://www.rte-france.com/en/eco2mix/co2-emissions] data.").Default("FRA").String()
 	globalEnergyMixDataUrl = "https://raw.githubusercontent.com/mlco2/codecarbon/master/codecarbon/data/private_infra/global_energy_mix.json"
 	globalEmissionFactor   = 475
-	getRteEnergyMixData    = GetRteEnergyMixData
+	getRteEnergyMixData    = utils.GetRteEnergyMixData
 )
 
 func init() {

collector/fixtures/e2e-test-cgroupsv1-output.txt → pkg/collector/fixtures/e2e-test-cgroupsv1-output.txt

@@ -45,8 +45,8 @@ batchjob_nvidia_gpu_jobid{uuid="GPU-61a65011-6571-a6d2-5ab8-66cbb6f7f9c3"} 11000
 batchjob_nvidia_gpu_jobid{uuid="GPU-f124aa59-d406-d45b-9481-8fcd694e6c9e"} 10000
 # HELP batchjob_rapl_package_joules_total Current RAPL package value in joules
 # TYPE batchjob_rapl_package_joules_total counter
-batchjob_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 258218.293244
-batchjob_rapl_package_joules_total{index="1",path="collector/fixtures/sys/class/powercap/intel-rapl:1"} 130570.505826
+batchjob_rapl_package_joules_total{index="0",path="pkg/collector/fixtures/sys/class/powercap/intel-rapl:0"} 258218.293244
+batchjob_rapl_package_joules_total{index="1",path="pkg/collector/fixtures/sys/class/powercap/intel-rapl:1"} 130570.505826
 # HELP batchjob_scrape_collector_duration_seconds batchjob_exporter: Duration of a collector scrape.
 # TYPE batchjob_scrape_collector_duration_seconds gauge
 # HELP batchjob_scrape_collector_success batchjob_exporter: Whether a collector succeeded.

collector/fixtures/e2e-test-cgroupsv2-output.txt → pkg/collector/fixtures/e2e-test-cgroupsv2-output.txt

@@ -45,8 +45,8 @@ batchjob_nvidia_gpu_jobid{uuid="GPU-61a65011-6571-a6d2-5ab8-66cbb6f7f9c3"} 11000
 batchjob_nvidia_gpu_jobid{uuid="GPU-f124aa59-d406-d45b-9481-8fcd694e6c9e"} 10000
 # HELP batchjob_rapl_package_joules_total Current RAPL package value in joules
 # TYPE batchjob_rapl_package_joules_total counter
-batchjob_rapl_package_joules_total{index="0",path="collector/fixtures/sys/class/powercap/intel-rapl:0"} 258218.293244
-batchjob_rapl_package_joules_total{index="1",path="collector/fixtures/sys/class/powercap/intel-rapl:1"} 130570.505826
+batchjob_rapl_package_joules_total{index="0",path="pkg/collector/fixtures/sys/class/powercap/intel-rapl:0"} 258218.293244
+batchjob_rapl_package_joules_total{index="1",path="pkg/collector/fixtures/sys/class/powercap/intel-rapl:1"} 130570.505826
 # HELP batchjob_scrape_collector_duration_seconds batchjob_exporter: Duration of a collector scrape.
 # TYPE batchjob_scrape_collector_duration_seconds gauge
 # HELP batchjob_scrape_collector_success batchjob_exporter: Whether a collector succeeded.

collector/helper.go → pkg/collector/helper.go

@@ -1,13 +1,9 @@
 package collector
 
 import (
-	"fmt"
 	"os"
-	"os/exec"
-	"regexp"
 
-	"github.com/go-kit/log"
-	"github.com/go-kit/log/level"
+	"regexp"
 )
 
 var (
@@ -37,13 +33,3 @@ func fileExists(filename string) bool {
 func SanitizeMetricName(metricName string) string {
 	return metricNameRegex.ReplaceAllString(metricName, "_")
 }
-
-// Execute command and return stdout/stderr
-func Execute(cmd string, args []string, logger log.Logger) ([]byte, error) {
-	level.Debug(logger).Log("msg", "Executing", "command", cmd, "args", fmt.Sprintf("%+v", args))
-	out, err := exec.Command(cmd, args...).CombinedOutput()
-	if err != nil {
-		err = fmt.Errorf("error running %s: %s", cmd, err)
-	}
-	return out, err
-}

collector/ipmi.go → pkg/collector/ipmi.go

@@ -16,6 +16,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+
+	utils "github.com/mahendrapaipuri/batchjob_monitoring/pkg/utils"
 )
 
 const ipmiCollectorSubsystem = "ipmi_dcmi"
@@ -71,7 +73,7 @@ func getValue(ipmiOutput []byte, regex *regexp.Regexp) (string, error) {
 // Update implements Collector and exposes IPMI DCMI power related metrics.
 func (c *impiCollector) Update(ch chan<- prometheus.Metric) error {
 	args := []string{""}
-	stdOut, err := Execute(*ipmiDcmiWrapperExec, args, c.logger)
+	stdOut, err := utils.Execute(*ipmiDcmiWrapperExec, args, c.logger)
 	if err != nil {
 		return err
 	}

collector/nvidia_gpus.go → pkg/collector/nvidia_gpus.go

@@ -12,6 +12,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+
+	utils "github.com/mahendrapaipuri/batchjob_monitoring/pkg/utils"
 )
 
 const nvidiaGpuJobMapCollectorSubsystem = "nvidia_gpu"
@@ -50,7 +52,7 @@ func init() {
 // NOTE: Hoping this command returns MIG devices too
 func getAllDevices(logger log.Logger) ([]Device, error) {
 	args := []string{"--query-gpu=name,uuid", "--format=csv"}
-	nvidiaSmiOutput, err := Execute("nvidia-smi", args, logger)
+	nvidiaSmiOutput, err := utils.Execute("nvidia-smi", args, logger)
 	if err != nil {
 		level.Error(logger).Log("msg", "nvidia-smi command to get list of devices failed", "err", err)
 		return nil, err

collector/slurm.go → pkg/collector/slurm.go

@@ -18,6 +18,8 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+
+	utils "github.com/mahendrapaipuri/batchjob_monitoring/pkg/utils"
 )
 
 const slurmCollectorSubsystem = "slurm_job"
@@ -311,7 +313,7 @@ func (c *slurmCollector) getJobLabels(jobid string) (string, string, string) {
 		} else {
 			fmt.Sscanf(string(content), "%s %s %s %s", &jobUid, &jobGid, &jobNodes, &jobWorkDir)
 		}
-		jobUuid, err = GetUuidFromString([]string{jobid, jobUid, jobGid, jobNodes, jobWorkDir})
+		jobUuid, err = utils.GetUuidFromString([]string{jobid, jobUid, jobGid, jobNodes, jobWorkDir})
 		if err != nil {
 			level.Error(c.logger).Log("msg", "Failed to generate UUID for job", "jobid", jobid, "err", err)
 			jobUuid = jobid
@@ -433,7 +435,7 @@ func (c *slurmCollector) getCgroupsV2Metrics(name string) (CgroupMetric, error)
 	level.Debug(c.logger).Log("msg", "Loading cgroup v2", "path", name)
 	// Files to parse out of the cgroup
 	controllers := []string{"cpu.stat", "memory.current", "memory.events", "memory.max", "memory.stat"}
-	data, err := LoadCgroupsV2Metrics(name, controllers)
+	data, err := utils.LoadCgroupsV2Metrics(name, *cgroupfsPath, controllers)
 	if err != nil {
 		level.Error(c.logger).Log("msg", "Failed to load cgroups", "path", name, "err", err)
 		metric.err = true

pkg/collector/types.go

@@ -0,0 +1,23 @@
+package collector
+
+type energyMixDataFields struct {
+	BioFuel                    float64 `json:"biofuel_TWh"`
+	CarbonIntensity            float64 `json:"carbon_intensity"`
+	Coal                       float64 `json:"coal_TWh"`
+	CountryName                string  `json:"country_name"`
+	Fossil                     float64 `json:"fossil_TWh"`
+	Gas                        float64 `json:"gas_TWh"`
+	Hydro                      float64 `json:"hydroelectricity_TWh"`
+	IsoCode                    string  `json:"iso_code"`
+	LowCarbon                  float64 `json:"low_carbon_TWh"`
+	Nuclear                    float64 `json:"nuclear_TWh"`
+	Oil                        float64 `json:"oil_TWh"`
+	OtherRenewable             float64 `json:"other_renewable_TWh"`
+	OtherRenewableExcluBioFuel float64 `json:"other_renewable_exc_biofuel_TWh"`
+	PerCapita                  float64 `json:"per_capita_Wh"`
+	Renewables                 float64 `json:"renewables_TWh"`
+	Solar                      float64 `json:"solar_TWh"`
+	Total                      float64 `json:"total_TWh"`
+	Wind                       float64 `json:"wind_TWh"`
+	Year                       int64   `json:"year"`
+}

collector/types.go → pkg/utils/types.go

@@ -1,4 +1,4 @@
-package collector
+package utils
 
 // Nicked from https://github.com/nmasse-itix/ego2mix
 type nationalRealTimeFields struct {
@@ -78,25 +78,3 @@ type nationalRealTimeResponse struct {
 	} `json:"parameters"`
 	Records []nationalRealTimeRecord `json:"records"`
 }
-
-type energyMixDataFields struct {
-	BioFuel                    float64 `json:"biofuel_TWh"`
-	CarbonIntensity            float64 `json:"carbon_intensity"`
-	Coal                       float64 `json:"coal_TWh"`
-	CountryName                string  `json:"country_name"`
-	Fossil                     float64 `json:"fossil_TWh"`
-	Gas                        float64 `json:"gas_TWh"`
-	Hydro                      float64 `json:"hydroelectricity_TWh"`
-	IsoCode                    string  `json:"iso_code"`
-	LowCarbon                  float64 `json:"low_carbon_TWh"`
-	Nuclear                    float64 `json:"nuclear_TWh"`
-	Oil                        float64 `json:"oil_TWh"`
-	OtherRenewable             float64 `json:"other_renewable_TWh"`
-	OtherRenewableExcluBioFuel float64 `json:"other_renewable_exc_biofuel_TWh"`
-	PerCapita                  float64 `json:"per_capita_Wh"`
-	Renewables                 float64 `json:"renewables_TWh"`
-	Solar                      float64 `json:"solar_TWh"`
-	Total                      float64 `json:"total_TWh"`
-	Wind                       float64 `json:"wind_TWh"`
-	Year                       int64   `json:"year"`
-}