diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..b21b961 --- /dev/null +++ b/.env.local @@ -0,0 +1,2 @@ +TF_WORKSPACE= +TF_VAR_PRIVATE_KEY= \ No newline at end of file diff --git a/.gitignore b/.gitignore index b337692..31ba4f3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ /cmd/indexing-service /indexing-service -*.car \ No newline at end of file +*.car +/build +deploy/.terraform +.env +.tfworkspace +/ucangen \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3d9ba5a --- /dev/null +++ b/Makefile @@ -0,0 +1,67 @@ +ifneq (,$(wildcard ./.env)) + include .env + export +else + $(error You haven't setup your .env file. Please refer to the readme) +endif +GOOS=linux +GOARCH=arm64 +GOCC?=go +GOFLAGS=-tags=lambda.norpc +CGO_ENABLED=0 +LAMBDAS=build/getclaims/bootstrap build/getroot/bootstrap build/notifier/bootstrap build/postclaims/bootstrap build/providercache/bootstrap build/remotesync/bootstrap + +ucangen: + go build -o ./ucangen cmd/ucangen/main.go + +.PHONY: ucankey + +ucankey: ucangen + ./ucangen + +.PHONY: clean-lambda + +clean-lambda: + rm -rf build + +.PHONY: clean-terraform + +clean-terraform: + tofu -chdir=deploy destroy + +.PHONY: clean + +clean: clean-lambda clean-terraform + +lambdas: $(LAMBDAS) + +.PHONY: $(LAMBDAS) + +$(LAMBDAS): build/%/bootstrap: + GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) $(GOCC) build $(GOFLAGS) -o $@ cmd/lambda/$*/main.go + +deploy/.terraform: + tofu -chdir=deploy init + +.tfworkspace: + tofu -chdir=deploy workspace new $(TF_WORKSPACE) + touch .tfworkspace + +.PHONY: init + +init: deploy/.terraform .tfworkspace + +.PHONY: validate + +validate: deploy/.terraform .tfworkspace + tofu -chdir=deploy validate + +.PHONY: plan + +plan: deploy/.terraform .tfworkspace $(LAMBDAS) + tofu -chdir=deploy plan + +.PHONY: apply + +apply: deploy/.terraform .tfworkspace $(LAMBDAS) + tofu -chdir=deploy apply diff --git a/README.md b/README.md index 5836e3f..cc8d450 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ * [Overview](#overview) * [Installation](#installation) +* [Deployment](#deployment) * [Contribute](#contribute) * [License](#license) @@ -23,6 +24,46 @@ $ go install github.com/storacha/indexing-service/cmd@latest ... ``` +## Deployment + +Deployment of this service to AWS is managed by terraform which you can invoke with make. + +### .env + +You need to first generate a .env with relevant vars. Copy `.env.local` to `.env` and then set the following environment variables: + +#### TF_WORKSPACE + +Best to set this to your name. "prod" and "staging" are reserved for shared deployments + +#### TF_VAR_private_key + +This is a multibase encoded ed25519 private key used to sign receipts and for the indexer's peer ID. For development, you can generate one by running `make ucankey` + +### Deployment commands + +Note that these commands will call needed prerequisites -- `make apply` will essentially do all of these start to finish. + +#### make lambdas + +This will simply compile the lambdas locally and put then in the `build` directory + +#### make init + +You should only need to run this once -- initializes your terraform deployment and workspace. Make sure you've set TF_WORKSPACE first! + +#### make validate + +This will validate your terraform configuration -- good to run to check errors in any changes you make to terraform configs + +#### make plan + +This will plan a deployment, but not execute it -- useful to see ahead what changes will happen when you run the next deployment + +#### make apply + +The big kahuna! This will deploy all of your changes, including redeploying lambdas if any of code changes + ## Contribute Early days PRs are welcome! diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000..056e6ce Binary files /dev/null and b/bootstrap differ diff --git a/cmd/ucangen/main.go b/cmd/ucangen/main.go new file mode 100644 index 0000000..31d740d --- /dev/null +++ b/cmd/ucangen/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "os" + + ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" +) + +func main() { + signer, err := ed25519.Generate() + if err != nil { + fmt.Println(err) + os.Exit(-1) + } + + asString, err := ed25519.Format(signer) + if err != nil { + fmt.Println(err) + os.Exit(-1) + } + + fmt.Printf("# %s\n", signer.DID().String()) + fmt.Println(asString) +} diff --git a/deploy/.terraform.lock.hcl b/deploy/.terraform.lock.hcl new file mode 100644 index 0000000..5c4975d --- /dev/null +++ b/deploy/.terraform.lock.hcl @@ -0,0 +1,37 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/archive" { + version = "2.6.0" + hashes = [ + "h1:s1OObC0b95ceQkrAMqL4q6wMYDBWYt8swZbLup+UJXI=", + "zh:046b3ba4223002d1cd1c917e8c21b58a636fcd751073745e3db99beebe254dd8", + "zh:1c1ed2ea0927b491689c3c7d178880cd9902f2a5339da8f46c56279920329a27", + "zh:1f17b47ba1bf18bd7bd30ea35c2ba32eaa23f8d08b3a35126edb31daf6ae10fd", + "zh:4b58aaac88335bb2ca482766e2682514fed78ff8cabe5665b6e5dd7c22ff9c81", + "zh:6c7dd6d4ff061d350fc6eb76866905c47450b8b8c1d2e238aa737afd48b6a267", + "zh:7b376916c5b911a3f887fd296c25ced36d8ba742b8482f1e0f092bf8fb008146", + "zh:8661139125b1ea7b89e0084377863dc820cdcbc433bb9a7c445350480f83b2c2", + "zh:e17c9056f210ec9a8c9cfe8a13ecd09ae59ad0a0197c96589b86eb4f7cf5326d", + "zh:ee15bddc7a596cccd400a762b6dadf1c8889faff7c931ae4b39f2e5404188da1", + "zh:f74355e6588daf88ec210d2967fbf5d22fa18c448d2807b8a7049dc777a2dbcb", + ] +} + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.72.1" + constraints = ">= 5.32.0" + hashes = [ + "h1:CEUIE9kJiDUQkVmeND4fpQCzXvCexHriL5lLuca70gM=", + "zh:02ee636137e5f8cc9d6900c55a3f3c85e99166b51d17cf96bd62b27182dc7449", + "zh:04877c5ce0a0fef6b355decbfe4941e7d5f22d2b7062cd87f70dafe845d635c7", + "zh:3d129024594dcf2edac180b8276decc946f4f33d653f44d3c04c4c28b3f85dab", + "zh:6fc7ecf746791211d64a38361ce12303dfc3ddf0e609e6d854bc8f3a7f242234", + "zh:8d65352eeba3fef611c90b5161336b0cccf3fed8dc2c710537d6578925e2b189", + "zh:9c99b31104c80d885aad1846e2b2f25371cbc9c23281fb0e213be0101a415f2c", + "zh:b220737d06dc8ef3a6aa32055c1c633d08c27e046b5fb730f93969ef11abb928", + "zh:b741b0e79001765c8d2ffdb569f70c0d8b877b870b660e573f3bb6f42dd55f28", + "zh:d2434f271f261ccd28a85aa15627ee9cfc9c319a48eb6c0aeb1ffaf80b6ede20", + "zh:df4b2338e3e89d66c1697fae9be378db94cb0d7d309c9dc537024cb755b7e21a", + ] +} diff --git a/deploy/dynamodb.tf b/deploy/dynamodb.tf new file mode 100644 index 0000000..c909556 --- /dev/null +++ b/deploy/dynamodb.tf @@ -0,0 +1,43 @@ +resource "aws_dynamodb_table" "metadata" { + name = "${terraform.workspace}-${var.app}-metadata" + billing_mode = "PAY_PER_REQUEST" + + attribute { + name = "provider" + type = "S" + } + + attribute { + name = "contextID" + type = "B" + } + + hash_key = "provider" + range_key = "contextID" + + tags = { + Name = "${terraform.workspace}-${var.app}-metadata" + } +} + +resource "aws_dynamodb_table" "chunk_links" { + name = "${terraform.workspace}-${var.app}-chunk-links" + billing_mode = "PAY_PER_REQUEST" + + attribute { + name = "provider" + type = "S" + } + + attribute { + name = "contextID" + type = "B" + } + + hash_key = "provider" + range_key = "contextID" + + tags = { + Name = "${terraform.workspace}-${var.app}-chunk-links" + } +} \ No newline at end of file diff --git a/deploy/elasticcache.tf b/deploy/elasticcache.tf new file mode 100644 index 0000000..26f0701 --- /dev/null +++ b/deploy/elasticcache.tf @@ -0,0 +1,84 @@ +locals { + caches = toset(["providers","indexes","claims"]) +} +resource "aws_kms_key" "cache_key" { + description = "KMS CMK for ${terraform.workspace} ${var.app}" + enable_key_rotation = true +} + +resource "aws_elasticache_serverless_cache" "cache" { + for_each = local.caches + + engine = "redis" + name = "${terraform.workspace}-${var.app}-${each.key}-cache" + cache_usage_limits { + data_storage { + maximum = terraform.workspace == "prod" ? 10 : 1 + unit = "GB" + } + ecpu_per_second { + maximum = terraform.workspace == "prod" ? 10000 : 1000 + } + } + daily_snapshot_time = "02:00" + description = "${terraform.workspace} ${var.app} ${each.key} serverless cluster" + kms_key_id = aws_kms_key.cache_key.arn + major_engine_version = "7" + security_group_ids = [aws_security_group.cache_security_group.id] + + snapshot_retention_limit = 7 + subnet_ids = aws_subnet.vpc_private_subnet[*].id + + user_group_id = aws_elasticache_user_group.cache_user_group.user_group_id +} + +resource "aws_elasticache_user_group" "cache_user_group" { + engine = "REDIS" + user_group_id = "${terraform.workspace}-${var.app}-redis" + + user_ids = [ + aws_elasticache_user.cache_default_user.id, + aws_elasticache_user.cache_iam_user.id + ] + + lifecycle { + ignore_changes = [user_ids] + } +} + +resource "aws_elasticache_user" "cache_default_user" { + user_id = "${terraform.workspace}-${var.app}-default-disabled" + user_name = "default" + access_string = "off ~keys* -@all +get" + authentication_mode { + type = "no-password-required" + } + lifecycle { + ignore_changes = [authentication_mode] + } + engine = "REDIS" +} + +resource "aws_elasticache_user" "cache_iam_user" { + user_id = "${terraform.workspace}-${var.app}-iam-user" + user_name = "${terraform.workspace}-${var.app}-iam-user" + access_string = "on ~* +@all" + authentication_mode { + type = "iam" + } + engine = "REDIS" +} + +resource "aws_security_group" "cache_security_group" { + + name = "${terraform.workspace}-${var.app}-cache-security-group" + description = "Security group for VPC access to redis" + vpc_id = aws_vpc.vpc.id + ingress { + cidr_blocks = [aws_vpc.vpc.cidr_block] + description = "Redis" + from_port = 6379 + to_port = 6379 + protocol = "tcp" + } +} \ No newline at end of file diff --git a/deploy/gateway.tf b/deploy/gateway.tf new file mode 100644 index 0000000..bd67a1e --- /dev/null +++ b/deploy/gateway.tf @@ -0,0 +1,134 @@ +resource "aws_apigatewayv2_api" "api" { + name = "${terraform.workspace}-${var.app}-api" + description = "${terraform.workspace} ${var.app} API Gateway" + protocol_type = "HTTP" +} + +resource "aws_apigatewayv2_route" "getclaims" { + api_id = aws_apigatewayv2_api.api.id + route_key = "GET /claims" + authorization_type = "NONE" + target = "integrations/${aws_apigatewayv2_integration.getclaims.id}" +} + +resource "aws_apigatewayv2_route" "getroot" { + api_id = aws_apigatewayv2_api.api.id + route_key = "GET /" + authorization_type = "NONE" + target = "integrations/${aws_apigatewayv2_integration.getroot.id}" +} + +resource "aws_apigatewayv2_route" "postclaims" { + api_id = aws_apigatewayv2_api.api.id + route_key = "POST /claims" + authorization_type = "NONE" + target = "integrations/${aws_apigatewayv2_integration.postclaims.id}" +} + + +resource "aws_apigatewayv2_integration" "getclaims" { + api_id = aws_apigatewayv2_api.api.id + integration_uri = aws_lambda_function.lambda["getclaims"].invoke_arn + payload_format_version = "2.0" + integration_type = "AWS_PROXY" + connection_type = "INTERNET" +} + + +resource "aws_apigatewayv2_integration" "getroot" { + api_id = aws_apigatewayv2_api.api.id + integration_uri = aws_lambda_function.lambda["getroot"].invoke_arn + payload_format_version = "2.0" + integration_type = "AWS_PROXY" + connection_type = "INTERNET" +} + + +resource "aws_apigatewayv2_integration" "postclaims" { + api_id = aws_apigatewayv2_api.api.id + integration_uri = aws_lambda_function.lambda["postclaims"].invoke_arn + payload_format_version = "2.0" + integration_type = "AWS_PROXY" + connection_type = "INTERNET" +} + +resource "aws_apigatewayv2_deployment" "deployment" { + depends_on = [aws_apigatewayv2_integration.getclaims, aws_apigatewayv2_integration.getroot, aws_apigatewayv2_integration.postclaims] + triggers = { + redeployment = sha1(join(",", [ + jsonencode(aws_apigatewayv2_integration.postclaims), + jsonencode(aws_apigatewayv2_integration.getclaims), + jsonencode(aws_apigatewayv2_integration.getroot), + jsonencode(aws_apigatewayv2_route.getclaims), + jsonencode(aws_apigatewayv2_route.getroot), + jsonencode(aws_apigatewayv2_route.postclaims), + ])) + } + + api_id = aws_apigatewayv2_api.api.id + description = "${terraform.workspace} ${var.app} API Deployment" + lifecycle { + create_before_destroy = true + } +} + +resource "aws_acm_certificate" "cert" { + domain_name = terraform.workspace == "prod" ? "${var.app}.storacha.network" : "${terraform.workspace}.${var.app}.storacha.network" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_zone" "primary" { + name = "${var.app}.storacha.network" +} + +resource "aws_route53_record" "cert_validation" { + allow_overwrite = true + name = tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_name + type = tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_type + zone_id = aws_route53_zone.primary.zone_id + records = [tolist(aws_acm_certificate.cert.domain_validation_options)[0].resource_record_value] + ttl = 60 +} + +resource "aws_acm_certificate_validation" "cert" { + certificate_arn = aws_acm_certificate.cert.arn + validation_record_fqdns = [ aws_route53_record.cert_validation.fqdn ] +} +resource "aws_apigatewayv2_domain_name" "custom_domain" { + domain_name = "${terraform.workspace}.${var.app}.storacha.network" + + domain_name_configuration { + certificate_arn = aws_acm_certificate_validation.cert.certificate_arn + endpoint_type = "REGIONAL" + security_policy = "TLS_1_2" + } +} +resource "aws_apigatewayv2_stage" "stage" { + api_id = aws_apigatewayv2_api.api.id + name = "$default" + lifecycle { + create_before_destroy = true + } +} + +resource "aws_apigatewayv2_api_mapping" "api_mapping" { + api_id = aws_apigatewayv2_api.api.id + stage = aws_apigatewayv2_stage.stage.id + domain_name = aws_apigatewayv2_domain_name.custom_domain.id +} + +resource "aws_route53_record" "api_gateway" { + zone_id = aws_route53_zone.primary.zone_id + name = "${terraform.workspace}.${var.app}.storacha.network" + type = "A" + + alias { + name = aws_apigatewayv2_domain_name.custom_domain.domain_name_configuration[0].target_domain_name + zone_id = aws_apigatewayv2_domain_name.custom_domain.domain_name_configuration[0].hosted_zone_id + evaluate_target_health = false + } +} \ No newline at end of file diff --git a/deploy/lambda.tf b/deploy/lambda.tf new file mode 100644 index 0000000..fe95dd4 --- /dev/null +++ b/deploy/lambda.tf @@ -0,0 +1,369 @@ +locals { + functions = { + getroot = { + name = "GETroot" + } + getclaims = { + name = "GETclaims" + } + postclaims = { + name = "POSTclaims" + } + notifier = { + name = "notifier" + } + providercache = { + name = "providercache" + } + remotesync = { + name = "remotesync" + } + } +} + +// zip the binary, as we can use only zip files to AWS lambda +data "archive_file" "function_archive" { + for_each = local.functions + + type = "zip" + source_file = "${path.root}/../build/${each.key}/bootstrap" + output_path = "${path.root}/../build/${each.key}/${each.key}.zip" +} + +# Define functions + +resource "aws_lambda_function" "lambda" { + depends_on = [ aws_cloudwatch_log_group.lambda_log_group ] + for_each = local.functions + + function_name = "${terraform.workspace}-${var.app}-lambda-${each.value.name}" + handler = "bootstrap" + runtime = "provided.al2023" + architectures = [ "arm64" ] + role = aws_iam_role.lambda_exec.arn + timeout = try(each.value.timeout, 3) + memory_size = try(each.value.memory_size, 128) + reserved_concurrent_executions = try(each.value.concurrency, -1) + source_code_hash = data.archive_file.function_archive[each.key].output_base64sha256 + filename = data.archive_file.function_archive[each.key].output_path # Path to your Lambda zip files + + environment { + variables = { + PROVIDERS_REDIS_URL = aws_elasticache_serverless_cache.cache["providers"].endpoint[0].address + PROVIDERS_REDIS_CACHE = aws_elasticache_serverless_cache.cache["providers"].name + INDEXES_REDIS_URL = aws_elasticache_serverless_cache.cache["indexes"].endpoint[0].address + INDEXES_REDIS_CACHE = aws_elasticache_serverless_cache.cache["indexes"].name + CLAIMS_REDIS_URL = aws_elasticache_serverless_cache.cache["claims"].endpoint[0].address + CLAIMS_REDIS_CACHE = aws_elasticache_serverless_cache.cache["claims"].name + REDIS_USER_ID = aws_elasticache_user.cache_iam_user.user_id + IPNI_ENDPOINT = "https://cid.contact" + PROVIDER_CACHING_QUEUE_URL = aws_sqs_queue.caching.id + PROVIDER_CACHING_BUCKET_NAME = aws_s3_bucket.caching_bucket.bucket + CHUNK_LINKS_TABLE_NAME = aws_dynamodb_table.chunk_links.id + METADATA_TABLE_NAME = aws_dynamodb_table.metadata.id + IPNI_STORE_BUCKET_NAME = aws_s3_bucket.ipni_store_bucket.bucket + NOTIFIER_HEAD_BUCKET_NAME = aws_s3_bucket.notifier_head_bucket.bucket + NOTIFIER_SNS_TOPIC_ARN = aws_sns_topic.published_advertisememt_head_change.id + PRIVATE_KEY = aws_ssm_parameter.private_key.name + IPNI_STORE_BUCKET_REGIONAL_DOMAIN = aws_s3_bucket.ipni_store_bucket.bucket_regional_domain_name + } + } + + vpc_config { + security_group_ids = [ + aws_security_group.lambda_security_group.id + ] + subnet_ids = aws_subnet.vpc_private_subnet[*].id + } +} + +# Acccess for the gateway + +resource "aws_lambda_permission" "api_gateway" { + for_each = aws_lambda_function.lambda + + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = each.value.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.api.execution_arn}/*/*" +} + +# Logging + +resource "aws_cloudwatch_log_group" "lambda_log_group" { + for_each = local.functions + name = "/aws/lambda/${terraform.workspace}-${var.app}-lambda-${each.value.name}" + retention_in_days = 7 + lifecycle { + prevent_destroy = false + } +} + +# Role policies and access to resources + +resource "aws_iam_role" "lambda_exec" { + name = "lambda_exec_role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "lambda_vpc_access_attachment" { + role = aws_iam_role.lambda_exec.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +data "aws_iam_policy_document" "lambda_elasticache_connect_document" { + + statement { + effect = "Allow" + actions = [ + "elasticache:Connect" + ] + + resources = [ + "arn:aws:elasticache:${var.region}:${var.allowed_account_ids[0]}:serverlesscache:${aws_elasticache_serverless_cache.cache["providers"].id}", + "arn:aws:elasticache:${var.region}:${var.allowed_account_ids[0]}:serverlesscache:${aws_elasticache_serverless_cache.cache["indexes"].id}", + "arn:aws:elasticache:${var.region}:${var.allowed_account_ids[0]}:serverlesscache:${aws_elasticache_serverless_cache.cache["claims"].id}", + "arn:aws:elasticache:${var.region}:${var.allowed_account_ids[0]}:user:${aws_elasticache_user.cache_iam_user.user_id}" + ] + } +} + +resource "aws_iam_policy" "lambda_elasticache_connect" { + name = "${terraform.workspace}-${var.app}-lambda-elasticache-connect" + policy = data.aws_iam_policy_document.lambda_elasticache_connect_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_elasticache_connect" { + policy_arn = aws_iam_policy.lambda_elasticache_connect.arn + role = aws_iam_role.lambda_exec.name +} + +data "aws_iam_policy_document" "lambda_dynamodb_put_get_document" { + statement { + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + ] + resources = [ + aws_dynamodb_table.chunk_links.arn, + aws_dynamodb_table.metadata.arn + ] + } +} + +resource "aws_iam_policy" "lambda_dynamodb_put_get" { + name = "${terraform.workspace}-${var.app}-lambda-dynamodb-put-get" + description = "This policy will be used by the lambda to put and get data from DynamoDB" + policy = data.aws_iam_policy_document.lambda_dynamodb_put_get_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_dynamodb_put_get" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_dynamodb_put_get.arn +} + + +data "aws_iam_policy_document" "lambda_s3_put_get_document" { + statement { + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = [ + aws_s3_bucket.caching_bucket.arn, + aws_s3_bucket.ipni_store_bucket.arn, + aws_s3_bucket.notifier_head_bucket.arn + ] + } +} + +resource "aws_iam_policy" "lambda_s3_put_get" { + name = "${terraform.workspace}-${var.app}-lambda-s3-put-get" + description = "This policy will be used by the lambda to put and get objects from S3" + policy = data.aws_iam_policy_document.lambda_s3_put_get_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_s3_put_get" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_s3_put_get.arn +} + +data "aws_iam_policy_document" "lambda_logs_document" { + statement { + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [for k, group in aws_cloudwatch_log_group.lambda_log_group : group.arn ] + } +} + +resource "aws_iam_policy" "lambda_logs" { + name = "${terraform.workspace}-${var.app}-lambda-logs" + description = "This policy will be used by the lambda to write logs" + policy = data.aws_iam_policy_document.lambda_logs_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_logs" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_logs.arn +} + +data "aws_iam_policy_document" "lambda_sns_document" { + statement { + actions = [ + "sns:Publish" + ] + resources = [ + aws_sns_topic.published_advertisememt_head_change.arn + ] + } +} + +resource "aws_iam_policy" "lambda_sns" { + name = "${terraform.workspace}-${var.app}-lambda-sns" + description = "This policy will be used by the lambda to push to sns" + policy = data.aws_iam_policy_document.lambda_sns_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_sns" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_sns.arn +} + +data "aws_iam_policy_document" "lambda_ssm_document" { + statement { + + effect = "Allow" + + actions = [ + "ssm:GetParameter", + ] + + resources = [ + aws_ssm_parameter.private_key.arn + ] + } +} + +resource "aws_iam_policy" "lambda_ssm" { + name = "${terraform.workspace}-${var.app}-lambda-ssm" + description = "This policy will be used by the lambda to access the parameter store" + policy = data.aws_iam_policy_document.lambda_ssm_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_ssm" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_ssm.arn +} + +data "aws_iam_policy_document" "lambda_sqs_document" { + statement { + + effect = "Allow" + + actions = [ + "sqs:SendMessage*", + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + + resources = [ + aws_sqs_queue.caching.arn + ] + } +} + +resource "aws_iam_policy" "lambda_sqs" { + name = "${terraform.workspace}-${var.app}-lambda-sqs" + description = "This policy will be used by the lambda to send messages to an SQS queue" + policy = data.aws_iam_policy_document.lambda_sqs_document.json +} + +resource "aws_iam_role_policy_attachment" "lambda_sqs" { + role = aws_iam_role.lambda_exec.name + policy_arn = aws_iam_policy.lambda_sqs.arn +} + +# event source mappings + +resource "aws_lambda_event_source_mapping" "event_source_mapping" { + event_source_arn = aws_sqs_queue.caching.arn + enabled = true + function_name = aws_lambda_function.lambda["providercache"].arn + batch_size = terraform.workspace == "prod" ? 10 : 1 +} + +resource "aws_cloudwatch_event_rule" "head_check" { + name = "${terraform.workspace}-${var.app}-lambda-head-check" + description = "Fires every minute" + schedule_expression = "cron(* * * * ? *)" +} + +resource "aws_cloudwatch_event_target" "notifier" { + rule = aws_cloudwatch_event_rule.head_check.name + target_id = "${terraform.workspace}-${var.app}-lambda-notifier-target" + arn = aws_lambda_function.lambda["notifier"].arn +} + +resource "aws_lambda_permission" "allow_cloudwatch_to_call_notifier" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda["notifier"].function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.head_check.arn +} + +resource "aws_sns_topic_subscription" "invoke_with_sns" { + topic_arn = aws_sns_topic.published_advertisememt_head_change.arn + protocol = "lambda" + endpoint = aws_lambda_function.lambda["remotesync"].arn +} + +resource "aws_lambda_permission" "allow_sns_invoke" { + statement_id = "AllowExecutionFromSNS" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda["remotesync"].function_name + principal = "sns.amazonaws.com" + source_arn = aws_sns_topic.published_advertisememt_head_change.arn +} + +# VPC Access + +resource "aws_security_group" "lambda_security_group" { + name = "${terraform.workspace}-${var.app}-lambda-security-group" + description = ("Allow traffic from lambda to elasticache") + vpc_id = aws_vpc.vpc.id + + egress { + from_port = 6379 + to_port = 6379 + protocol = "tcp" + description = "Allow elasticache access" + security_groups = [ + aws_security_group.cache_security_group.id, + ] + } + + egress { + from_port = 443 + to_port = 443 + protocol = "tcp" + description = "Allow internet access" + cidr_blocks = ["0.0.0.0/0"] + } +} \ No newline at end of file diff --git a/deploy/main.tf b/deploy/main.tf new file mode 100644 index 0000000..907e3b9 --- /dev/null +++ b/deploy/main.tf @@ -0,0 +1,37 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.32.0" + } + archive = { + source = "hashicorp/archive" + } + } + backend "s3" { + bucket = "storacha-terraform-state" + key = "storacha/indexing-service/terraform.tfstate" + region = "us-west-2" + } +} + +provider "aws" { + region = var.region + allowed_account_ids = var.allowed_account_ids + default_tags { + + tags = { + "Environment" = terraform.workspace + "ManagedBy" = "OpenTofu" + Owner = "storacha" + Team = "Storacha Engineer" + Organization = "Storacha" + Project = "${var.app}" + } + } +} + +provider "aws" { + alias = "virginia" + region = "us-east-1" +} \ No newline at end of file diff --git a/deploy/s3.tf b/deploy/s3.tf new file mode 100644 index 0000000..9a81c48 --- /dev/null +++ b/deploy/s3.tf @@ -0,0 +1,50 @@ +resource "aws_s3_bucket" "caching_bucket" { + bucket = "${terraform.workspace}-${var.app}-caching-bucket" +} + +resource "aws_s3_bucket" "ipni_store_bucket" { + bucket = "${terraform.workspace}-${var.app}-ipni-store-bucket" +} + +resource "aws_s3_bucket_public_access_block" "ipni_store_bucket" { + bucket = aws_s3_bucket.ipni_store_bucket.id + + block_public_acls = true + block_public_policy = false + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_cors_configuration" "ipni_store_cors" { + bucket = aws_s3_bucket.ipni_store_bucket.bucket + + cors_rule { + allowed_headers = ["*"] + allowed_methods = ["GET", "HEAD"] + allowed_origins = ["*"] + expose_headers = ["Content-Length", "Content-Type", "Content-MD5", "ETag"] + max_age_seconds = 86400 + } +} + +resource "aws_s3_bucket_policy" "ipni_store_policy" { + depends_on = [ aws_s3_bucket_public_access_block.ipni_store_bucket ] + bucket = aws_s3_bucket.ipni_store_bucket.id + + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "PublicRead", + "Effect" : "Allow", + "Principal" : "*", + "Action" : ["s3:GetObject", "s3:GetObjectVersion"], + "Resource" : ["${aws_s3_bucket.ipni_store_bucket.arn}/*"] + } + ] + }) +} + +resource "aws_s3_bucket" "notifier_head_bucket" { + bucket = "${terraform.workspace}-${var.app}-notifier-head-bucket" +} \ No newline at end of file diff --git a/deploy/secrets.tf b/deploy/secrets.tf new file mode 100644 index 0000000..951ddc4 --- /dev/null +++ b/deploy/secrets.tf @@ -0,0 +1,10 @@ +resource "aws_ssm_parameter" "private_key" { + name = "/${var.app}/${terraform.workspace}/Secret/PRIVATE_KEY/value" + description = "private key for the deployed environment" + type = "SecureString" + value = var.private_key + + tags = { + environment = "production" + } +} \ No newline at end of file diff --git a/deploy/sns.tf b/deploy/sns.tf new file mode 100644 index 0000000..1e2781a --- /dev/null +++ b/deploy/sns.tf @@ -0,0 +1,3 @@ +resource "aws_sns_topic" "published_advertisememt_head_change" { + name = "${terraform.workspace}-${var.app}-published-advertisement-head-change" +} diff --git a/deploy/sqs.tf b/deploy/sqs.tf new file mode 100644 index 0000000..63ddbe4 --- /dev/null +++ b/deploy/sqs.tf @@ -0,0 +1,29 @@ + + +resource "aws_sqs_queue" "caching" { + name = "${terraform.workspace}-${var.app}-caching.fifo" + fifo_queue = true + content_based_deduplication = true + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.caching_deadletter.arn + maxReceiveCount = 4 + }) + tags = { + Name = "${terraform.workspace}-${var.app}-caching" + } +} + +resource "aws_sqs_queue" "caching_deadletter" { + fifo_queue = true + content_based_deduplication = true + name = "${terraform.workspace}-${var.app}-caching-deadletter.fifo" +} + +resource "aws_sqs_queue_redrive_allow_policy" "caching" { + queue_url = aws_sqs_queue.caching_deadletter.id + + redrive_allow_policy = jsonencode({ + redrivePermission = "byQueue", + sourceQueueArns = [aws_sqs_queue.caching.arn] + }) +} \ No newline at end of file diff --git a/deploy/variables.tf b/deploy/variables.tf new file mode 100644 index 0000000..7275a06 --- /dev/null +++ b/deploy/variables.tf @@ -0,0 +1,21 @@ +variable "app" { + description = "The name of the application" + type = string + default = "indexer" +} + +variable "region" { + description = "aws region for all services" + type = string + default = "us-west-2" +} + +variable "allowed_account_ids" { + description = "account ids used for AWS" + type = list(string) + default = ["505595374361"] +} +variable private_key { + description = "private_key for the peer for this deployment" + type = string +} \ No newline at end of file diff --git a/deploy/vpc.tf b/deploy/vpc.tf new file mode 100644 index 0000000..4736aaf --- /dev/null +++ b/deploy/vpc.tf @@ -0,0 +1,121 @@ +locals { + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) +} + +data "aws_availability_zones" "available" {} + + +resource "aws_vpc" "vpc" { + cidr_block = local.vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + tags = { + Name = "${terraform.workspace}-${var.app}-vpc" + } +} + +resource "aws_internet_gateway" "vpc_internet_gateway" { + vpc_id = aws_vpc.vpc.id + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-internet-gateway" + } +} + +resource "aws_subnet" "vpc_public_subnet" { + count = length(local.azs) + + vpc_id = aws_vpc.vpc.id + availability_zone = local.azs[count.index] + cidr_block = cidrsubnet(local.vpc_cidr, 8, count.index) + + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-public-subnet-${local.azs[count.index]}" + } +} + +resource "aws_subnet" "vpc_private_subnet" { + count = length(local.azs) + + vpc_id = aws_vpc.vpc.id + availability_zone = local.azs[count.index] + cidr_block = cidrsubnet(local.vpc_cidr, 8, count.index+10) + + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-private-subnet-${local.azs[count.index]}" + } +} + +resource "aws_route_table" "vpc_public_route_table" { + count = length(local.azs) + vpc_id = aws_vpc.vpc.id + + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-public-route-table-${local.azs[count.index]}" + } +} + +resource "aws_route_table" "vpc_private_route_table" { + count = length(local.azs) + vpc_id = aws_vpc.vpc.id + + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-private-route-table-${local.azs[count.index]}" + } +} + +resource "aws_route_table_association" "vpc_public_route_table_association" { + count = length(local.azs) + + subnet_id = aws_subnet.vpc_public_subnet[count.index].id + route_table_id = aws_route_table.vpc_public_route_table[count.index].id +} + +resource "aws_route_table_association" "vpc_private_route_table_association" { + count = length(local.azs) + + subnet_id = aws_subnet.vpc_private_subnet[count.index].id + route_table_id = aws_route_table.vpc_private_route_table[count.index].id +} + +resource "aws_route" "vpc_public_internet_gateway" { + count = length(local.azs) + + route_table_id = aws_route_table.vpc_public_route_table[count.index].id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.vpc_internet_gateway.id + + timeouts { + create = "5m" + } +} + +resource "aws_route" "vpc_private_nat_gateway" { + count = length(local.azs) + + route_table_id = aws_route_table.vpc_private_route_table[count.index].id + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.vpc_nat[count.index].id + + timeouts { + create = "5m" + } +} + +resource "aws_eip" "vpc_elastic_ip" { + count = length(local.azs) + domain = "vpc" + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-elastic-ip-${local.azs[count.index]}" + } +} + +resource "aws_nat_gateway" "vpc_nat" { + count = length(local.azs) + subnet_id = aws_subnet.vpc_public_subnet[count.index].id + allocation_id = aws_eip.vpc_elastic_ip[count.index].id + depends_on = [aws_internet_gateway.vpc_internet_gateway] + tags = { + Name = "${terraform.workspace}-${var.app}-vpc-nat-${local.azs[count.index]}" + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 1a084a1..08a007e 100644 --- a/go.mod +++ b/go.mod @@ -56,9 +56,9 @@ require ( github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.36.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 github.com/aws/aws-sdk-go-v2/service/sns v1.33.2 github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2 + github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index a44bbc6..2193d24 100644 --- a/go.sum +++ b/go.sum @@ -83,12 +83,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3 h1:xxHGZ+wUgZNACQmxtdvP5tgzfsxGS3vPpTP5Hy3iToE= github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 h1:Rrqru2wYkKQCS2IM5/JrgKUQIoNTqA6y/iuxkjzxC6M= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2/go.mod h1:QuCURO98Sqee2AXmqDNxKXYFm2OEDAVAPApMqO0Vqnc= github.com/aws/aws-sdk-go-v2/service/sns v1.33.2 h1:GeVRrB1aJsGdXxdPY6VOv0SWs+pfdeDlKgiBxi0+V6I= github.com/aws/aws-sdk-go-v2/service/sns v1.33.2/go.mod h1:c6Sj8zleZXYs4nyU3gpDKTzPWu7+t30YUXoLYRpbUvU= github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2 h1:kmbcoWgbzfh5a6rvfjOnfHSGEqD13qu1GfTPRZqg0FI= github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2/go.mod h1:/UPx74a3M0WYeT2yLQYG/qHhkPlPXd6TsppfGgy2COk= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU= +github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= diff --git a/pkg/aws/iamauthtokenrequest.go b/pkg/aws/iamauthtokenrequest.go new file mode 100644 index 0000000..5909e75 --- /dev/null +++ b/pkg/aws/iamauthtokenrequest.go @@ -0,0 +1,109 @@ +package aws + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + signer "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +// Taken from: https://github.com/redis/go-redis/discussions/2343 + +const ( + REQUEST_PROTOCOL = "http://" + PARAM_ACTION = "Action" + PARAM_USER = "User" + ACTION_NAME = "connect" + SERVICE_NAME = "elasticache" + PARAM_EXPIRES = "X-Amz-Expires" + TOKEN_EXPIRY_SECONDS = 899 + + EMPTY_BODY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" // the hex encoded SHA-256 of an empty string +) + +type IAMAuthTokenRequest struct { + userID string + cacheName string + region string +} + +func (i *IAMAuthTokenRequest) toSignedRequestUri(ctx context.Context, credential aws.Credentials) (string, error) { + req, err := i.getSignableRequest() + if err != nil { + return "", err + } + + signedURI, _, err := i.sign(ctx, req, credential) + if err != nil { + return "", err + } + + u, err := url.Parse(signedURI) + if err != nil { + return "", err + } + + res := url.URL{ + Scheme: "http", + Host: u.Host, + Path: "/", + RawQuery: u.RawQuery, + } + + return strings.Replace(res.String(), REQUEST_PROTOCOL, "", 1), nil +} + +func (i *IAMAuthTokenRequest) getSignableRequest() (*http.Request, error) { + query := url.Values{ + PARAM_ACTION: {ACTION_NAME}, + PARAM_USER: {i.userID}, + PARAM_EXPIRES: {strconv.FormatInt(int64(TOKEN_EXPIRY_SECONDS), 10)}, + } + + signURL := url.URL{ + Scheme: "http", + Host: i.cacheName, + Path: "/", + RawQuery: query.Encode(), + } + + req, err := http.NewRequest(http.MethodGet, signURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +func (i *IAMAuthTokenRequest) sign(ctx context.Context, req *http.Request, credential aws.Credentials) (signedURI string, signedHeaders http.Header, err error) { + s := signer.NewSigner() + return s.PresignHTTP(ctx, credential, req, EMPTY_BODY_SHA256, SERVICE_NAME, i.region, time.Now()) +} + +func redisCredentialVerifier(cfg aws.Config, userID string, cacheName string) func(context.Context) (string, string, error) { + return func(ctx context.Context) (string, string, error) { + iamAuthTokenRequest := IAMAuthTokenRequest{ + userID: userID, + cacheName: cacheName, + region: cfg.Region, + } + + credentials, err := cfg.Credentials.Retrieve(ctx) + if err != nil { + return "", "", fmt.Errorf("getting aws credentials: %w", err) + } + iamAuthToken, err := iamAuthTokenRequest.toSignedRequestUri(ctx, credentials) + + if err != nil { + return "", "", fmt.Errorf("attempting to obtain signed redis loging: %w", err) + } + + return userID, iamAuthToken, nil + } +} diff --git a/pkg/aws/service.go b/pkg/aws/service.go index 6d61b42..f3eb4ab 100644 --- a/pkg/aws/service.go +++ b/pkg/aws/service.go @@ -8,8 +8,9 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" "github.com/redis/go-redis/v9" "github.com/storacha/go-ucanto/principal" ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" @@ -50,17 +51,18 @@ func FromEnv(ctx context.Context) Config { if err != nil { panic(fmt.Errorf("loading aws default config: %w", err)) } - secretsClient := secretsmanager.NewFromConfig(awsConfig) - response, err := secretsClient.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(mustGetEnv("PRIVATE_KEY")), + ssmClient := ssm.NewFromConfig(awsConfig) + response, err := ssmClient.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(mustGetEnv("PRIVATE_KEY")), + WithDecryption: aws.Bool(true), }) if err != nil { panic(fmt.Errorf("retrieving private key: %w", err)) } - if response.SecretString == nil { + if response.Parameter == nil || response.Parameter.Value == nil { panic(ErrNoPrivateKey) } - id, err := ed25519.Parse(*response.SecretString) + id, err := ed25519.Parse(*response.Parameter.Value) if err != nil { panic(fmt.Errorf("parsing private key: %s", err)) } @@ -74,25 +76,31 @@ func FromEnv(ctx context.Context) Config { ipniStoreKeyPrefix = "/ipni/v1/ad/" } + peer, err := peer.IDFromPrivateKey(cryptoPrivKey) + if err != nil { + panic(fmt.Errorf("parsing private key to peer: %w", err)) + } + + ipniPublisherAnnounceAddress := fmt.Sprintf("/dns4/%s/tcp/443/https/p2p/%s", mustGetEnv("IPNI_STORE_BUCKET_REGIONAL_DOMAIN"), peer.String()) return Config{ Config: awsConfig, Signer: id, ServiceConfig: construct.ServiceConfig{ PrivateKey: cryptoPrivKey, ProvidersRedis: redis.Options{ - Addr: mustGetEnv("PROVIDER_REDIS_URL"), - Password: mustGetEnv("PROVIDER_REDIS_PASSWD"), + Addr: mustGetEnv("PROVIDERS_REDIS_URL"), + CredentialsProviderContext: redisCredentialVerifier(awsConfig, mustGetEnv("REDIS_USER_ID"), mustGetEnv("PROVIDERS_REDIS_CACHE")), }, ClaimsRedis: redis.Options{ - Addr: mustGetEnv("CLAIMS_REDIS_URL"), - Password: mustGetEnv("CLAIMS_REDIS_PASSWD"), + Addr: mustGetEnv("CLAIMS_REDIS_URL"), + CredentialsProviderContext: redisCredentialVerifier(awsConfig, mustGetEnv("REDIS_USER_ID"), mustGetEnv("CLAIMS_REDIS_CACHE")), }, IndexesRedis: redis.Options{ - Addr: mustGetEnv("INDEXES_REDIS_URL"), - Password: mustGetEnv("INDEXES_REDIS_PASSWD"), + Addr: mustGetEnv("INDEXES_REDIS_URL"), + CredentialsProviderContext: redisCredentialVerifier(awsConfig, mustGetEnv("REDIS_USER_ID"), mustGetEnv("INDEXES_REDIS_CACHE")), }, IndexerURL: mustGetEnv("IPNI_ENDPOINT"), - PublisherAnnounceAddrs: []string{mustGetEnv("IPNI_PUBLISHER_ANNOUNCE_ADDRESS")}, + PublisherAnnounceAddrs: []string{ipniPublisherAnnounceAddress}, }, SQSCachingQueueURL: mustGetEnv("PROVIDER_CACHING_QUEUE_URL"), CachingBucket: mustGetEnv("PROVIDER_CACHING_BUCKET_NAME"), diff --git a/pkg/service/queryresult/datamodel/queryresult.ipldsch b/pkg/service/queryresult/datamodel/queryresult.ipldsch index eced1e4..80d9d3b 100644 --- a/pkg/service/queryresult/datamodel/queryresult.ipldsch +++ b/pkg/service/queryresult/datamodel/queryresult.ipldsch @@ -1,6 +1,6 @@ type QueryResult union { | QueryResult0_1 "index/query/result@0.1" -} +} representation keyed type QueryResult0_1 struct { claims optional [Link]