Migrate from api-frontend.

.github/workflows/ci.yaml

@@ -0,0 +1,34 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v3
+
+    - name: Get dependencies
+      run: go mod download
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Run unit tests
+      run: go test $(go list ./... | grep -v /integration) -cover -race -v
+
+    - name: Run integration tests
+      run: go test ./integration-tests

.github/workflows/ent-ci.yaml

@@ -0,0 +1,19 @@
+name: EntCI
+on:
+  push:
+  # Run whenever code is changed in the master.
+    branches:
+      - main
+  # Run on PRs where something changed under the `ent/` directory.
+  pull_request:
+    paths:
+      - 'ent/*'
+jobs:
+  ent:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+      - uses: ent/contrib/ci@master

.github/workflows/logging-presubmit.yml

@@ -0,0 +1,13 @@
+name: semgrep
+on:
+  push:
+    branches:
+      - "*"
+jobs:
+  semgrep-ci:
+    runs-on: ubuntu-20.04
+    container:
+      image: semgrep/semgrep
+    steps:
+      - uses: actions/checkout@v3
+      - run: semgrep ci --config .semgrep.yml --exclude='main.go' --exclude='server/server.go' --exclude='logging/*' --

.github/workflows/migration-ci.yaml

@@ -0,0 +1,37 @@
+name: Atlas CI
+on:
+  # Run whenever code is changed in the main branch,
+  # change this to your root branch.
+  # push:
+    # branches:
+    #   - main
+  # Run on PRs where something changed under the `ent/migrate/migrations/` directory.
+  pull_request:
+    paths:
+      - 'ent/migrate/migrations/*'
+jobs:
+  lint:
+    services:
+      # Spin up a postgres:10 container to be used as the dev-database for analysis.
+      postgres:
+        image: postgres:10
+        env:
+          POSTGRES_DB: test
+          POSTGRES_PASSWORD: pass
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0 # Mandatory unless "latest" is set below.
+      - uses: ariga/atlas-action@v0
+        with:
+          dir: ent/migrate/migrations
+          dir-format: golang-migrate # Or: atlas, goose, dbmate
+          dev-url: postgres://postgres:pass@localhost:5432/test?sslmode=disable

.gitignore

@@ -0,0 +1,5 @@
+tmp/
+.vscode/*
+.vscode/settings.json
+.idea/*
+.DS_Store

Dockerfile

@@ -0,0 +1,28 @@
+# Start from the official Go image to build the binary.
+# Use the same version of Go as your project (1.21.5).
+FROM golang:1.21.5 AS builder
+
+# Set the working directory inside the container.
+WORKDIR /go/src/app
+
+# Copy the Go Modules manifests and download modules to leverage Docker cache.
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy the rest of the source code.
+COPY . .
+
+# Build the Go app.
+# -o /go/bin/app: Compile the binary to /go/bin/app.
+# You might need to adjust the build command depending on your project's structure.
+RUN CGO_ENABLED=0 GOOS=linux go build -v -o /go/bin/app
+
+# Use a small base image to create a minimal final image.
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates
+
+# Copy the pre-built binary file from the previous stage.
+COPY --from=builder /go/bin/app /go/bin/app
+
+# Run the binary.
+ENTRYPOINT ["/go/bin/app"]

Dockerfile.integration

@@ -0,0 +1,19 @@
+# Used for integration tests. See docker-compose-integration.yml.
+# Start from the official Go image.
+FROM golang:1.21.5
+
+# Set the working directory inside the container.
+WORKDIR /go/src/app
+
+# Copy the Go Modules manifests and the whole source code.
+COPY go.mod go.sum ./
+COPY . .
+
+# Download Go modules.
+RUN go mod download
+
+# Compile the application to ensure it builds.
+RUN CGO_ENABLED=0 GOOS=linux go build -v -o /app/bin
+
+# The default command runs the app, but can be overridden in docker-compose.
+CMD ["/app/bin"]

README.md

@@ -0,0 +1,178 @@
+# registry-backend
+
+The first service to receive API requests
+
+## Local Dev
+
+### Golang
+
+https://go.dev/doc/install
+
+### Supabase
+
+Install [Supabase Cli](https://supabase.com/docs/guides/cli/getting-started)
+
+`brew install supabase/tap/supabase`
+
+`supabase start`
+
+Open [Supabase Studio](http://127.0.0.1:54323/project/default) locally.
+
+### Start API Server
+
+`docker compose up`
+
+This commands starts the server with Air that listens to changes. It connects to the Supabase running locally.
+
+### Set up local ADC credentials
+
+These are needed for authenticating Firebase JWT token auth + calling other GCP APIs.
+
+When testing login with registry, use this:
+`gcloud config set project dreamboothy-dev`
+
+When testing workspace / VM creation, use this:
+`gcloud config set project dreamboothy`
+
+`gcloud auth application-default login`
+
+If you are testing creating a node, you need to impersonate a service account because it requires signing cloud storage urls.
+
+`gcloud auth application-default login --impersonate-service-account [email protected]`
+
+TODO(robinhuang): Create a service account suitable for dev.
+
+# Code Generation
+
+Make sure you install the golang packages locally.
+
+`go get`
+
+## Schema Change
+
+Update the files in `ent/schema`.
+
+### Regenerate code
+
+This should search all directories and run go generate. This will run all the commands in the `generate.go` files in the repository.
+
+`go generate ./...`
+
+Or manually run:
+
+`go run -mod=mod entgo.io/ent/cmd/ent generate --feature sql/upsert --feature sql/lock ./ent/schema`
+
+## API Spec Change (openapi.yml)
+
+### Regenerate code
+
+This should search all directories and run go generate. This will run all the commands in the `generate.go` files in the repository.
+
+`go generate ./...`
+
+Or manually run:
+
+`export PATH="$PATH:$HOME/bin:$HOME/go/bin"`
+
+https://github.com/deepmap/oapi-codegen/issues/795
+
+`oapi-codegen --config drip/codegen.yaml openapi.yml`
+
+## TroubleShooting / Common Errors
+
+Here are some common errors and how to resolve them.
+
+### Firebase Token Errors
+
+Usually in localdev, we use dreamboothy-dev Firebase project for authentication. This conflicts with our machine creation logic because all of those machine images are in dreamboothy. TODO(robinhuang): Figure out a solution for this. Either we replicate things in dreamboothy-dev, or we pass project information separately when creating machine images.
+
+### Creating VM instance error:
+
+**Example:**
+
+```
+{
+    "severity": "ERROR",
+    "error": "error creating instance: Post \"https://compute.googleapis.com/compute/v1/projects/dreamboothy/zones/us-central1-a/instances\": oauth2: \"invalid_grant\" \"reauth related error (invalid_rapt)\" \"https://support.google.com/a/answer/9368756\"",
+    "time": "2024-02-26T01:32:27Z",
+    "message": "Error creating instance:"
+}
+
+{
+    "severity": "ERROR",
+    "error": "failed to get session using author id 'nz0vAxfqWLSrqPcUhspyuOEp03z2': error creating instance: Post \"https://compute.googleapis.com/compute/v1/projects/dreamboothy/zones/us-central1-a/instances\": oauth2: \"invalid_grant\" \"reauth related error (invalid_rapt)\" \"https://support.google.com/a/answer/9368756\"",
+    "time": "2024-02-26T01:32:27Z",
+    "message": "Error occurred Path: /workflows/:id, Method: GET\n"
+}
+```
+
+**Resolution:**
+
+You would likely need to run `gcloud auth application-default login` again and
+restart your docker containers/services to pick up the new credentials.
+
+### Calling CreateSession endpoint
+
+Use the postman collection to call the CreateSession endpoint. You should be able to import changes with `openapi.yml`
+file.
+You should use this as a request body since there are list of supported GPU type.
+
+```json
+{
+  "gpu-type": "nvidia-tesla-t4"
+}
+```
+
+### Bypass Authentication Error
+
+In order to bypass authentication error, you can add make the following changes in `firebase_auth.go` file.
+
+```go
+package drip_middleware
+
+func FirebaseMiddleware(entClient *ent.Client) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(ctx echo.Context) error {
+			userDetails := &UserDetails{
+				ID:    "test-james-token-id",
+				Email: "[email protected]",
+				Name:  "James",
+			}
+
+			authdCtx := context.WithValue(ctx.Request().Context(), UserContextKey, userDetails)
+			ctx.SetRequest(ctx.Request().WithContext(authdCtx))
+			newUserError := db.UpsertUser(ctx.Request().Context(), entClient, userDetails.ID, userDetails.Email, userDetails.Name)
+			if newUserError != nil {
+				log.Ctx(ctx).Info().Ctx(ctx.Request().Context()).Err(newUserError).Msg("error User upserted successfully.")
+			}
+			return next(ctx)
+		}
+	}
+}
+
+```
+
+### Machine Image Not Found
+
+We use a custom machine image to create VM instances. That machine image is specified in `docker-compose.yml` file.
+
+```yaml
+MACHINE_IMAGE: "comfy-cloud-template-3"
+```
+
+If you are getting an error that the machine image is not found, you can create a new machine image by following the
+steps below:
+
+**TODO**: explore steps to create machine image with comfy setup.
+
+For the purpose of just testing endpoints, you don't really need to worry about Comfy specific machine image.
+You can simply create a new VM on the GCP console and use that VM's image to create a new machine image.
+And then update the `docker-compose.yml` file with the new machine image name.
+
+## Clean Up Resources
+
+You can use this script to cleanup resources for specific user.
+
+```shell
+`docker compose -f scripts/cleanup/docker-compose.cleanup.yml run --rm cleanup -u <user id>`
+```

app.yaml

@@ -0,0 +1,14 @@
+runtime: go121
+env: standard
+
+instance_class: F1  # Default instance class. Consider changing it based on your needs
+
+handlers:
+  - url: /.*
+    script: auto
+    secure: always  # Optional: Redirects HTTP to HTTPS.
+
+automatic_scaling:
+  min_idle_instances: automatic  # Default is automatic (spins down completely)
+  max_idle_instances: automatic  # Default is automatic
+  min_instances: 0  # Default is 0

cloudbuild.yaml

@@ -0,0 +1,20 @@
+steps:
+  # build the container image
+  - name: "gcr.io/cloud-builders/docker"
+    args: ["build", "-t", "us-central1-docker.pkg.dev/dreamboothy/registry-backend/registry-backend-image:$SHORT_SHA", "."]
+    # push container image
+  - name: "gcr.io/cloud-builders/docker"
+    args: ["push", "us-central1-docker.pkg.dev/dreamboothy/registry-backend/registry-backend-image:$SHORT_SHA"]
+  # Publish the release
+  - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:458.0.1'
+    entrypoint: 'bash'
+    args:
+    - '-c'
+    - > 
+      gcloud deploy releases create release-registry-backend-$SHORT_SHA
+      --project=dreamboothy
+      --region=us-central1
+      --delivery-pipeline=comfy-backend-api-pipeline
+      --images=registry-backend-image-substitute=us-central1-docker.pkg.dev/dreamboothy/registry-backend/registry-backend-image:$SHORT_SHA
+options:
+  machineType: 'E2_HIGHCPU_8'