Merge branch 'main' into remove-pluggable-executors

.cspell/custom-dictionary.txt

@@ -355,6 +355,7 @@ wasmlogs
 wasmmodels
 wazero
 wdbaruni's
+simonwo
 webui
 wesbos
 winderresearch
@@ -371,3 +372,63 @@ yyyymmddhhmm
 zarr
 zerolog
 zidane
+IMDC
+kvstore
+unmarshalling
+Nowf
+pkey
+machineid
+bacerror
+Nacked
+pqueue
+Routez
+Connz
+Subsz
+nuid
+Noticef
+Warnf
+Debugf
+Tracef
+sresource
+Syncer
+mathgo
+providables
+JSONV
+Idxs
+boltdblib
+hclog
+THAMTShard
+mergo
+serde
+qdisc
+puuid
+pkgs
+pscbin
+rocm
+strg
+otlploggrpc
+yacspin
+APITLSCA
+APITLSCA
+Milli
+Errf
+doesn
+cicd
+nvme
+fdisk
+mdstat
+xcom
+Fooco
+Eventuallyf
+Truef
+sekret
+Equalf
+Doesnt
+HAMT
+dagpb
+Berdly
+frrist
+swaggo
+isbadactor
+installationid
+firstbacalhauimage

.golangci.yml

@@ -54,12 +54,6 @@ linters-settings:
       - shadow
   lll:
     line-length: 140
-  misspell:
-    locale: US
-    ignore-words:
-    - favour
-    - cancelled
-    - cancelling
   nolintlint:
     allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
     allow-unused: true # report any unused nolint directives
@@ -89,7 +83,6 @@ linters:
   - govet
   - ineffassign
   - lll
-  - misspell
   - mnd
   - nakedret
   - noctx

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   - id: detect-aws-credentials
     args: [--allow-missing-credentials]
   - id: detect-private-key
-    exclude: testdata/.*
+    exclude: 'testdata/.*|test-integration/certificates/.*'
   - id: check-yaml
   - id: check-json
 - repo: https://github.com/astral-sh/ruff-pre-commit

DESIGN.md

@@ -103,16 +103,11 @@ Ideally, we will also allow much more fine-grained control, specifying location,
   - She has a file `process.py` which includes the python code necessary to execute in a function called 'downscale()' which takes a file handle to local, processes it, and returns a bytestream.
   - She executes the following command:
 ```
-ifps job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+ipfs job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
 ```
   - This runs the command in a local executor, first installing all the python packages necessary, and then executing them, on the subset of data available on that node.
   - Once complete, the system returns the CID of the updated dataset that she can download.
 
-- **SCENARIO 3** Want to burst to cloud but cannot move entire dataset in short time
-  - DHASH CAN YOU HELP FLESH OUT
-  - **PUSH COMPUTE INTO GENE SEQUENCER**
-  - **PIPE TO S3**
-
 ## Components to Build
 
 - Build an application that listens for jobs over NATS, receives payment somehow, runs the job in {kubernetes, docker, idk}, and returns the result to the use (ideally the 'result' is in the form of an ipfs object and we can just return the hash).
@@ -125,13 +120,3 @@ ifps job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrz
 Bacalhau means cod (the fish) in Portuguese (where several folks were brainstorming this topic).
 
 Compute-Over-Data == Cod == Bacalhau
-
-## Prior Art / Parallel Projects
-* IPFS-FAN - distributed serverless - https://research.protocol.ai/publications/ipfs-fan-a-function-addressable-computation-network/delarocha2021a.pdf
-* IPLS : A Framework for Decentralized Federated Learning- https://arxiv.org/pdf/2101.01901v1.pdf
-* Interplanetary Distributed Computing (2018) - https://github.com/yenkuanlee/IPDC
-* IPTF - IPFS + TensorFlow (2018) - https://github.com/tesserai/iptf
-* Lurk -> Run queries over Filecoin Sealed Data (no public paper yet)
-* Radix - Nomad based scheduler for  IPFS cluster (only) - high level spec doc https://docs.google.com/document/d/18hdYBmDlvusEOQ-iSNIO_IAEOvJVFL1MyAU_B8hON9Q/edit?usp=sharing
-* Bringing Arbitrary Compute to Authoritative Data https://queue.acm.org/detail.cfm?id=2645649
-* Manta: a scalable, distributed object store https://github.com/joyent/manta

Makefile

@@ -469,5 +469,5 @@ release: build-bacalhau
 	cp bin/bacalhau .
 
 .PHONY: spellcheck-code
-spellcheck-code:  ## Runs a spellchecker over all code - MVP just does one file
-	cspell -c .cspell-code.json lint ./pkg/authn/**
+spellcheck-code:
+	cspell lint  -c cspell.yaml --quiet "**/*.{go,js,ts,jsx,tsx,md,yml,yaml,json}"

clients/python/docs/OrchestratorApi.md

@@ -35,7 +35,7 @@ from pprint import pprint
 api_instance = bacalhau_apiclient.OrchestratorApi()
 id = 'id_example' # str | ID to get the job for
 include = 'include_example' # str | Takes history and executions as options. If empty will not include anything else. (optional)
-limit = 56 # int | Number of history or exeuctions to fetch. Should be used in conjugation with include (optional)
+limit = 56 # int | Number of history or executions to fetch. Should be used in conjugation with include (optional)
 
 try:
     # Returns a job.
@@ -47,11 +47,11 @@ except ApiException as e:
 
 ### Parameters
 
-Name | Type | Description  | Notes
-------------- | ------------- | ------------- | -------------
- **id** | **str**| ID to get the job for | 
- **include** | **str**| Takes history and executions as options. If empty will not include anything else. | [optional] 
- **limit** | **int**| Number of history or exeuctions to fetch. Should be used in conjugation with include | [optional] 
+Name | Type | Description                                                                          | Notes
+------------- | ------------- |--------------------------------------------------------------------------------------| -------------
+ **id** | **str**| ID to get the job for                                                                | 
+ **include** | **str**| Takes history and executions as options. If empty will not include anything else.    | [optional] 
+ **limit** | **int**| Number of history or executions to fetch. Should be used in conjugation with include | [optional] 
 
 ### Return type
 

cmd/cli/agent/alive_test.go

@@ -30,7 +30,7 @@ func (s *AliveSuite) TestAliveJSONOutput() {
 
 	aliveInfo := &apimodels.IsAliveResponse{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &aliveInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", err)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", err)
 	s.Require().True(aliveInfo.IsReady())
 }
 
@@ -40,6 +40,6 @@ func (s *AliveSuite) TestAliveYAMLOutput() {
 
 	aliveInfo := &apimodels.IsAliveResponse{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &aliveInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().True(aliveInfo.IsReady())
 }

cmd/cli/agent/node_test.go

@@ -28,7 +28,7 @@ func (s *NodeSuite) TestNodeJSONOutput() {
 
 	nodeInfo := &models.NodeState{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &nodeInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", out)
 	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in json.")
 }
 
@@ -38,6 +38,6 @@ func (s *NodeSuite) TestNodeYAMLOutput() {
 
 	nodeInfo := &models.NodeState{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &nodeInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in yaml.")
 }

cmd/cli/agent/version_test.go

@@ -45,7 +45,7 @@ func (s *VersionSuite) TestVersionJSONOutput() {
 	expectedVersion := version.Get()
 	printedVersion := &models.BuildVersionInfo{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &printedVersion)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", out)
 	s.Require().Equal(expectedVersion, printedVersion, "Versions do not match in json.")
 }
 
@@ -58,6 +58,6 @@ func (s *VersionSuite) TestVersionYAMLOutput() {
 	expectedVersion := version.Get()
 	printedVersion := &models.BuildVersionInfo{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &printedVersion)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().Equal(expectedVersion, printedVersion, "Versions do not match in yaml.")
 }

cmd/cli/docker/docker_run_cli_test.go

@@ -1,5 +1,7 @@
 //go:build unit || !integration
 
+/* spell-checker: disable */
+
 package docker
 
 import (
@@ -431,7 +433,7 @@ func TestJobFlagParsing(t *testing.T) {
 			},
 			expectedError: false,
 		},
-		// TODO(forrest): if/when validtion on the network config is adjusted expect this test to fail.
+		// TODO(forrest): if/when validation on the network config is adjusted expect this test to fail.
 		{
 			name:  "with none network and domains",
 			flags: []string{"--network=none", "--domain=example.com", "--domain=example.io", "image:tag"},
@@ -487,30 +489,30 @@ func TestJobFlagParsing(t *testing.T) {
 		},
 		{
 			name:  "with s3 publisher",
-			flags: []string{"--publisher=s3://mybucket/mykey", "image:tag"},
+			flags: []string{"--publisher=s3://myBucket/myKey", "image:tag"},
 			assertJob: func(t *testing.T, j *models.Job) {
 				defaultJobAssertions(t, j)
 				task := j.Task()
 				s3publisher, err := publisher_s3.DecodePublisherSpec(task.Publisher)
 				require.NoError(t, err)
 				assert.Equal(t, publisher_s3.PublisherSpec{
-					Bucket: "mybucket",
-					Key:    "mykey",
+					Bucket: "myBucket",
+					Key:    "myKey",
 				}, s3publisher)
 			},
 			expectedError: false,
 		},
 		{
 			name:  "with s3 publisher with opts",
-			flags: []string{"-p=s3://mybucket/mykey,opt=region=us-west-2,opt=endpoint=https://s3.custom.com", "image:tag"},
+			flags: []string{"-p=s3://myBucket/myKey,opt=region=us-west-2,opt=endpoint=https://s3.custom.com", "image:tag"},
 			assertJob: func(t *testing.T, j *models.Job) {
 				defaultJobAssertions(t, j)
 				task := j.Task()
 				s3publisher, err := publisher_s3.DecodePublisherSpec(task.Publisher)
 				require.NoError(t, err)
 				assert.Equal(t, publisher_s3.PublisherSpec{
-					Bucket:   "mybucket",
-					Key:      "mykey",
+					Bucket:   "myBucket",
+					Key:      "myKey",
 					Region:   "us-west-2",
 					Endpoint: "https://s3.custom.com",
 				}, s3publisher)
@@ -519,15 +521,15 @@ func TestJobFlagParsing(t *testing.T) {
 		},
 		{
 			name:  "with s3 publisher with options",
-			flags: []string{"-p=s3://mybucket/mykey,option=region=us-west-2,option=endpoint=https://s3.custom.com", "image:tag"},
+			flags: []string{"-p=s3://myBucket/myKey,option=region=us-west-2,option=endpoint=https://s3.custom.com", "image:tag"},
 			assertJob: func(t *testing.T, j *models.Job) {
 				defaultJobAssertions(t, j)
 				task := j.Task()
 				s3publisher, err := publisher_s3.DecodePublisherSpec(task.Publisher)
 				require.NoError(t, err)
 				assert.Equal(t, publisher_s3.PublisherSpec{
-					Bucket:   "mybucket",
-					Key:      "mykey",
+					Bucket:   "myBucket",
+					Key:      "myKey",
 					Region:   "us-west-2",
 					Endpoint: "https://s3.custom.com",
 				}, s3publisher)

cmd/cli/docker/docker_run_test.go

@@ -165,12 +165,12 @@ func (s *DockerRunSuite) TestRun_SubmitUrlInputs() {
 		{inputURL: InputURL{url: "https://raw.githubusercontent.com/bacalhau-project/bacalhau/main/main.go", pathInContainer: "/inputs", filename: "main.go", flag: "-i"}},
 	}
 
-	for _, turls := range testURLs {
+	for _, urls := range testURLs {
 		ctx := context.Background()
 		flagsArray := []string{"docker", "run"}
 
-		flagsArray = append(flagsArray, turls.inputURL.flag, turls.inputURL.url)
-		flagsArray = append(flagsArray, "ubuntu", "cat", fmt.Sprintf("%s/%s", turls.inputURL.pathInContainer, turls.inputURL.filename))
+		flagsArray = append(flagsArray, urls.inputURL.flag, urls.inputURL.url)
+		flagsArray = append(flagsArray, "ubuntu", "cat", fmt.Sprintf("%s/%s", urls.inputURL.pathInContainer, urls.inputURL.filename))
 
 		_, out, err := s.ExecuteTestCobraCommand(flagsArray...)
 		s.Require().NoError(err, "Error submitting job")
@@ -180,8 +180,8 @@ func (s *DockerRunSuite) TestRun_SubmitUrlInputs() {
 		s.Require().Equal(1, len(j.Task().InputSources), "Number of job urls != # of test urls.")
 		urlSpec, err := storage_url.DecodeSpec(j.Task().InputSources[0].Source)
 		s.Require().NoError(err)
-		s.Require().Equal(turls.inputURL.url, urlSpec.URL, "Test URL not equal to URL from job.")
-		s.Require().Equal(turls.inputURL.pathInContainer, j.Task().InputSources[0].Target, "Test Path not equal to Path from job.")
+		s.Require().Equal(urls.inputURL.url, urlSpec.URL, "Test URL not equal to URL from job.")
+		s.Require().Equal(urls.inputURL.pathInContainer, j.Task().InputSources[0].Target, "Test Path not equal to Path from job.")
 
 	}
 }
@@ -252,8 +252,8 @@ func (s *DockerRunSuite) TestRun_SubmitWorkdir() {
 	}{
 		{workdir: "", errorCode: 0},
 		{workdir: "/", errorCode: 0},
-		{workdir: "./mydir", errorCode: 1},
-		{workdir: "../mydir", errorCode: 1},
+		{workdir: "./myDir", errorCode: 1},
+		{workdir: "../myDir", errorCode: 1},
 		{workdir: "http://foo.com", errorCode: 1},
 		{workdir: "/foo//", errorCode: 0}, // double forward slash is allowed in unix
 		{workdir: "/foo//bar", errorCode: 0},

cmd/cli/exec/exec.go

@@ -228,7 +228,7 @@ func PrepareJob(cmd *cobra.Command, cmdArgs []string, unknownArgs []string, opti
 	job.Task().Env = options.TaskSettings.EnvironmentVariables
 	job.Task().InputSources = options.TaskSettings.InputSources.Values()
 	if err != nil {
-		return nil, fmt.Errorf("parsing job labes: %w", err)
+		return nil, fmt.Errorf("parsing job labels: %w", err)
 	}
 	job.Constraints, err = options.JobSettings.Constraints()
 	if err != nil {