From d8cff15dad513da531382c0158cb27d33eccf9a6 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Mon, 1 Jul 2024 15:38:39 -0700 Subject: [PATCH 01/15] BTS and summit UWS will be handled by phalanx. --- applications/uws/values-base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/applications/uws/values-base.yaml b/applications/uws/values-base.yaml index c9ef439fe2..90eaa67877 100644 --- a/applications/uws/values-base.yaml +++ b/applications/uws/values-base.yaml @@ -53,3 +53,4 @@ atocps: pullPolicy: Always env: RUN_ARG: 1 + From 927c25c879fc037200b31f230f77073b10c61b44 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 09:35:57 -0700 Subject: [PATCH 02/15] BTS: Kafka and Butler updates for nublado. --- applications/nublado/secrets-base.yaml | 13 +++++++++++++ applications/nublado/values-base.yaml | 17 +++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 applications/nublado/secrets-base.yaml diff --git a/applications/nublado/secrets-base.yaml b/applications/nublado/secrets-base.yaml new file mode 100644 index 0000000000..4c2ea47752 --- /dev/null +++ b/applications/nublado/secrets-base.yaml @@ -0,0 +1,13 @@ +"aws-credentials.ini": + description: >- + S3 Butler credentials in AWS format. + onepassword: + encoded: true +"kafka_credentials.txt": + description: >- + Kafka credentials for the telescope control system Kafka broker. +"postgres-credentials.txt": + description: >- + PostgreSQL credentials in its pgpass format for the oods database. + onepassword: + encoded: true diff --git a/applications/nublado/values-base.yaml b/applications/nublado/values-base.yaml index b312027c26..f6198672df 100644 --- a/applications/nublado/values-base.yaml +++ b/applications/nublado/values-base.yaml @@ -8,19 +8,25 @@ controller: numReleases: 0 numWeeklies: 3 numDailies: 2 - cycle: 37 - recommendedTag: "recommended_c0037" + cycle: null + recommendedTag: "recommended_k0001" lab: extraAnnotations: k8s.v1.cni.cncf.io/networks: "kube-system/dds" env: + AWS_SHARED_CRDENTIALS_FILE: "/opt/lsst/software/jupyterlab/secrets/aws-credentials.ini" DAF_BUTLER_REPOSITORY_INDEX: "/project/data-repos.yaml" LSST_DDS_INTERFACE: "net1" LSST_DDS_PARTITION_PREFIX: "base" LSST_SITE: "base" + LSST_TOPIC_SUBNAME: "sal" + LSST_KAFKA_PASSFILE: "/opt/lsst/software/jupyterlab/secrets/kafka_credentials.txt" + LSST_KAFKA_BROKER_ADDR: "sasquatch-kafka-brokers.sasquatch:9092" + LSST_SCHEMA_REGISTRY_URL: "http://sasquatch-schema-registry.sasquatch:8081" PGPASSFILE: "/opt/lsst/software/jupyterlab/secrets/postgres-credentials.txt" PGUSER: "oods" RSP_SITE_TYPE: "telescope" + S3_ENDPOINT_URL: "https://s3-butler.ls.lsst.org" initContainers: - name: "inithome" image: @@ -32,6 +38,10 @@ controller: volumeName: "home" pullSecret: "pull-secret" secrets: + - secretName: "nublado-lab-secret" + secretKey: "aws-credentials.ini" + - secretName: "nublado-lab-secret" + secretKey: "kafka_credentials.txt" - secretName: "nublado-lab-secret" secretKey: "postgres-credentials.txt" volumes: @@ -91,3 +101,6 @@ jupyterhub: db: upgrade: true url: "postgresql://nublado3@postgresdb01.ls.lsst.org/nublado3" + +secrets: + templateSecrets: true From 657ee968be35a9232d99c002d5c19b364f0a130e Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 09:51:31 -0700 Subject: [PATCH 03/15] BTS: Updates to sasquatch for Kafka control system. --- applications/sasquatch/values-base.yaml | 77 ++++++++++++------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/applications/sasquatch/values-base.yaml b/applications/sasquatch/values-base.yaml index c889c3fb4b..e38013b5a5 100644 --- a/applications/sasquatch/values-base.yaml +++ b/applications/sasquatch/values-base.yaml @@ -15,7 +15,14 @@ strimzi-kafka: limits: cpu: 4 memory: 8Gi + cluster: + monitorLabel: + lsst.io/monitor: "true" kafka: + disruption_tolerance: 1 + config: + auto.create.topics.enable: false + log.cleaner.min.compaction.lag.ms: 259200000 storage: storageClassName: rook-ceph-block size: 1Ti @@ -35,6 +42,25 @@ strimzi-kafka: host: sasquatch-base-kafka-1.lsst.codes - loadBalancerIP: "139.229.153.68" host: sasquatch-base-kafka-2.lsst.codes + resources: + requests: + memory: 80Gi + cpu: 4 + limits: + memory: 80Gi + cpu: 4 + metricsConfig: + enabled: true + kafkaExporter: + enabled: true + enableSaramaLogging: true + resources: + requests: + cpu: 200m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi kraft: enabled: true kafkaController: @@ -59,6 +85,17 @@ strimzi-kafka: enabled: true consdb: enabled: true + registry: + ingress: + enabled: true + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 + hostname: base-lsp.lsst.codes + path: /schema-registry(/|$)(.*) + connect: + config: + key.converter: org.apache.kafka.connect.json.JsonConverter + key.converter.schemas.enable: false influxdb: persistence: @@ -82,9 +119,6 @@ kafka-connect-manager: enabled: true topicsRegex: "lsst.sal.MTMount" tasksMax: "8" - comcam: - enabled: true - topicsRegex: "lsst.sal.CCCamera|lsst.sal.CCHeaderService|lsst.sal.CCOODS" eas: enabled: true topicsRegex: "lsst.sal.DIMM|lsst.sal.DSM|lsst.sal.EPM|lsst.sal.ESS|lsst.sal.HVAC|lsst.sal.WeatherForecast" @@ -107,12 +141,6 @@ kafka-connect-manager: test: enabled: true topicsRegex: "lsst.sal.Test" - pmd: - enabled: true - topicsRegex: "lsst.sal.PMD" - calsys: - enabled: true - topicsRegex: "lsst.sal.ATMonochromator|lsst.sal.ATWhiteLight|lsst.sal.CBP|lsst.sal.Electrometer|lsst.sal.FiberSpectrograph|lsst.sal.LinearStage|lsst.sal.TunableLaser" mtaircompressor: enabled: true topicsRegex: "lsst.sal.MTAirCompressor" @@ -159,16 +187,6 @@ telegraf-kafka-consumer: timestamp_field: "private_efdStamp" topicRegexps: | [ "lsst.sal.MTMount" ] - comcam: - enabled: true - replicaCount: 1 - interval: "1s" - flush_interval: "1s" - union_mode: "nullable" - timestamp_format: "unix" - timestamp_field: "private_efdStamp" - topicRegexps: | - [ "lsst.sal.CCCamera", "lsst.sal.CCHeaderService", "lsst.sal.CCOODS" ] eas: enabled: true replicaCount: 1 @@ -239,26 +257,6 @@ telegraf-kafka-consumer: timestamp_field: "private_efdStamp" topicRegexps: | [ "lsst.sal.Test" ] - pmd: - enabled: true - replicaCount: 1 - interval: "1s" - flush_interval: "1s" - union_mode: "nullable" - timestamp_format: "unix" - timestamp_field: "private_efdStamp" - topicRegexps: | - [ "lsst.sal.PMD" ] - calsys: - enabled: true - replicaCount: 1 - interval: "1s" - flush_interval: "1s" - union_mode: "nullable" - timestamp_format: "unix" - timestamp_field: "private_efdStamp" - topicRegexps: | - [ "lsst.sal.ATMonochromator", "lsst.sal.ATWhiteLight", "lsst.sal.CBP", "lsst.sal.Electrometer", "lsst.sal.FiberSpectrograph", "lsst.sal.LinearStage", "lsst.sal.TunableLaser" ] mtaircompressor: enabled: true replicaCount: 1 @@ -301,6 +299,7 @@ telegraf-kafka-consumer: [ "lsst.sal.MTCamera", "lsst.sal.MTHeaderService", "lsst.sal.MTOODS" ] kafdrop: + cmdArgs: "--message.format=AVRO --message.keyFormat=DEFAULT --topic.deleteEnabled=false --topic.createEnabled=false" ingress: enabled: true hostname: base-lsp.lsst.codes From 8afc9f2ffe148c128451724001fe09aa739615d2 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 11:02:20 -0700 Subject: [PATCH 04/15] BTS: Add control system general config to environment. --- environments/values-base.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/environments/values-base.yaml b/environments/values-base.yaml index 51e647e30a..4369a1eb04 100644 --- a/environments/values-base.yaml +++ b/environments/values-base.yaml @@ -17,3 +17,8 @@ applications: strimzi: true telegraf: true telegraf-ds: true + +controlSystem: + imageTag: "k0001" + siteTag: "base" + s3EndpointUrl: "https://s3.ls.lsst.org" From ff3ad4a7512b61fc66e12f40252b9356e7e36908 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 11:32:58 -0700 Subject: [PATCH 05/15] BTS: Add auxtel configuration. --- applications/auxtel/secrets.yaml | 3 + applications/auxtel/values-base.yaml | 146 +++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 applications/auxtel/values-base.yaml diff --git a/applications/auxtel/secrets.yaml b/applications/auxtel/secrets.yaml index 0885edfff9..bc8c08a65a 100644 --- a/applications/auxtel/secrets.yaml +++ b/applications/auxtel/secrets.yaml @@ -13,3 +13,6 @@ ts-salkafka-password: postgres-credentials.txt: description: >- Credentials for oods user Postgres access. + copy: + application: nublado + key: postgres-credentials.txt diff --git a/applications/auxtel/values-base.yaml b/applications/auxtel/values-base.yaml new file mode 100644 index 0000000000..c7350107ab --- /dev/null +++ b/applications/auxtel/values-base.yaml @@ -0,0 +1,146 @@ +ataos: + image: + repository: ts-dockerhub.lsst.org/ataos + pullPolicy: Always + +atdome-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/atdome + pullPolicy: Always + env: + RUN_ARG: --simulate + +atdometrajectory: + image: + repository: ts-dockerhub.lsst.org/atdometrajectory + pullPolicy: Always + +atheaderservice: + image: + repository: ts-dockerhub.lsst.org/headerservice + pullPolicy: Always + env: + URL_SPEC: --lfa_mode s3 --s3instance ls + TSTAND_HEADERSERVICE: BASE + CAMERA: at + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + +athexapod-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/athexapod + pullPolicy: Always + +atmcs-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/atmcs_sim + pullPolicy: Always + +atoods: + image: + repository: ts-dockerhub.lsst.org/atoods + pullPolicy: Always + env: + DAF_BUTLER_REPOSITORY_INDEX: /project/data-repos.yaml + CTRL_OODS_CONFIG_FILE: /etc/atoods.yaml + butlerSecret: + containerPath: /home/saluser/.lsst + dbUser: oods + secretFilename: postgres-credentials.txt + nfsMountpoint: + - name: auxtel-gen3-butler + containerPath: /repo/LATISS + readOnly: false + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/repo/LATISS + - name: auxtel-oods-data + containerPath: /data + readOnly: false + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel + configfile: + path: /etc + filename: atoods.yaml + content: | + defaultInterval: &interval + days: 0 + hours: 0 + minutes: 0 + seconds: 0 + + ingester: + imageStagingDirectory: /data/staging/auxtel/oods + butlers: + - butler: + instrument: lsst.obs.lsst.Latiss + class: + import : lsst.ctrl.oods.gen3ButlerIngester + name : Gen3ButlerIngester + stagingDirectory : /data/lsstdata/BTS/auxtel/oods/gen3butler/raw + badFileDirectory: /data/lsstdata/BTS/auxtel/oods/gen3butler/badfiles + repoDirectory : /repo/LATISS + collections: + - LATISS/raw/all + scanInterval: + <<: *interval + hours: 1 + filesOlderThan: + <<: *interval + days: 30 + batchSize: 20 + scanInterval: + <<: *interval + seconds: 2 + + cacheCleaner: + # ONLY clean out empty directories here, never files + clearEmptyDirectories: + - /data/lsstdata/BTS/auxtel/oods/gen3butler/raw + # clean out empty directories and old files from these directories + clearEmptyDirectoriesAndOldFiles: + - /data/lsstdata/BTS/auxtel/oods/gen3butler/badfiles + - /data/staging/auxtel/oods + - /data/staging/auxtel/forwarder + scanInterval: + <<: *interval + hours: 1 + filesOlderThan: + <<: *interval + days: 31 + directoriesEmptyForMoreThan: + <<: *interval + days: 2 + +atpneumatics-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/at_pneumatics_sim + pullPolicy: Always + +atptg: + image: + repository: ts-dockerhub.lsst.org/ptkernel + pullPolicy: Always + env: + TELESCOPE: AT + +atspectrograph-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/atspec + pullPolicy: Always + env: + RUN_ARG: --simulate + +hexapod-sim: + enabled: true From ff21a296a80456e51341d8bf4bf64d49351e670f Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 11:35:22 -0700 Subject: [PATCH 06/15] BTS: Add calsys configuration. --- applications/calsys/values-base.yaml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 applications/calsys/values-base.yaml diff --git a/applications/calsys/values-base.yaml b/applications/calsys/values-base.yaml new file mode 100644 index 0000000000..93879ad422 --- /dev/null +++ b/applications/calsys/values-base.yaml @@ -0,0 +1,40 @@ +gcheaderservice1: + enabled: true + image: + repository: ts-dockerhub.lsst.org/headerservice + pullPolicy: Always + env: + CAMERA: gc1 + TSTAND_HEADERSERVICE: BASE + URL_SPEC: --lfa_mode s3 --s3instance ls + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + +simulation-gencam: + enabled: true + classifier: genericcamera1 + image: + repository: ts-dockerhub.lsst.org/genericcamera + pullPolicy: Always + env: + RUN_ARG: 1 + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + service: + enabled: true + port: 5013 + type: LoadBalancer From 18410d831530ec720a0d4b631c21af2ced7c05af Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 12:00:15 -0700 Subject: [PATCH 07/15] BTS: Add control-system-test configuration. --- applications/control-system-test/values-base.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 applications/control-system-test/values-base.yaml diff --git a/applications/control-system-test/values-base.yaml b/applications/control-system-test/values-base.yaml new file mode 100644 index 0000000000..e7453656ef --- /dev/null +++ b/applications/control-system-test/values-base.yaml @@ -0,0 +1,11 @@ +test42: + image: + repository: ts-dockerhub.lsst.org/test + pullPolicy: Always + env: + RUN_ARG: 42 + +integration-testing: + enabled: true + envEfd: base_efd + s3Bucket: rubinobs-lfa-ls From d4b46b991e40d37e4db11a617571c2373d6841d4 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 12:15:02 -0700 Subject: [PATCH 08/15] BTS: Add eas configuration. --- applications/eas/values-base.yaml | 334 ++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 applications/eas/values-base.yaml diff --git a/applications/eas/values-base.yaml b/applications/eas/values-base.yaml new file mode 100644 index 0000000000..5a3a9c3a69 --- /dev/null +++ b/applications/eas/values-base.yaml @@ -0,0 +1,334 @@ +auxtel-ess201-sim: + enabled: true + classifier: ess201 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 201 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +auxtel-ess202-sim: + enabled: true + classifier: ess202 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 202 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +auxtel-ess203-sim: + enabled: true + classifier: ess203 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 203 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +auxtel-ess204-sim: + enabled: true + classifier: ess204 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 204 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +auxtel-ess205-sim: + enabled: true + classifier: ess205 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 205 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +calibhill-ess301-sim: + enabled: true + classifier: ess301 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 301 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +dimm1-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/dimm + pullPolicy: Always + env: + RUN_ARG: 1 --simulate + resources: + requests: + cpu: 23m + memory: 107Mi + limits: + cpu: 230m + memory: 1070Mi + +dimm2-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/dimm + pullPolicy: Always + env: + RUN_ARG: 2 --simulate + resources: + requests: + cpu: 23m + memory: 107Mi + limits: + cpu: 230m + memory: 1070Mi + +dsm1-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/dsm + pullPolicy: Always + env: + CSC_INDEX: 1 + RUN_ARG: --simulate 1 --state enabled + resources: + requests: + cpu: 14m + memory: 90Mi + limits: + cpu: 140m + memory: 900Mi + +dsm2-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/dsm + pullPolicy: Always + env: + CSC_INDEX: 2 + RUN_ARG: --simulate 2 --state enabled + resources: + requests: + cpu: 10m + memory: 90Mi + limits: + cpu: 100m + memory: 900Mi + +epm1-sim: + enabled: true + classifier: epm1 + image: + repository: ts-dockerhub.lsst.org/epm + pullPolicy: Always + env: + RUN_ARG: 1 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +m2-ess106-sim: + enabled: true + classifier: ess106 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 106 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +mtdome-ess101-sim: + enabled: true + classifier: ess101 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 101 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +mtdome-ess102-sim: + enabled: true + classifier: ess102 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 102 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +mtdome-ess103-sim: + enabled: true + classifier: ess103 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 103 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +mtdome-ess107-sim: + enabled: true + classifier: ess107 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 107 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +mtdome-ess108-sim: + enabled: true + classifier: ess108 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 108 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +tma-ess001-sim: + enabled: true + classifier: ess1 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 1 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +tma-ess104-sim: + enabled: true + classifier: ess104 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 104 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +tma-ess105-sim: + enabled: true + classifier: ess105 + image: + repository: ts-dockerhub.lsst.org/ess + pullPolicy: Always + env: + RUN_ARG: 105 --simulate + resources: + requests: + cpu: 19m + memory: 90Mi + limits: + cpu: 190m + memory: 900Mi + +weatherforecast: + image: + repository: ts-dockerhub.lsst.org/weatherforecast + pullPolicy: Always + env: + RUN_ARG: --state enabled + envSecrets: + - name: METEOBLUE_API_KEY + key: meteoblue-api-key + resources: + requests: + cpu: 9m + memory: 95Mi + limits: + cpu: 90m + memory: 950Mi From ec2d36ef7f75a6b45643f49e3bc2a13cd537617e Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 12:29:36 -0700 Subject: [PATCH 09/15] BTS: Add love configuration. --- applications/love/values-base.yaml | 324 +++++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 applications/love/values-base.yaml diff --git a/applications/love/values-base.yaml b/applications/love/values-base.yaml new file mode 100644 index 0000000000..c4d4f4219e --- /dev/null +++ b/applications/love/values-base.yaml @@ -0,0 +1,324 @@ +love-commander: + image: + repository: ts-dockerhub.lsst.org/love-commander + pullPolicy: Always + env: + S3_INSTANCE: ls + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + service: + enabled: true + port: 5000 + type: ClusterIP + +love-manager: + manager: + frontend: + image: + repository: ts-dockerhub.lsst.org/love-manager + pullPolicy: Always + env: + SERVER_URL: base-lsp.lsst.codes + OLE_API_HOSTNAME: base-lsp.lsst.codes + AUTH_LDAP_1_SERVER_URI: ldap://ipa1.ls.lsst.org + AUTH_LDAP_2_SERVER_URI: ldap://ipa2.ls.lsst.org + AUTH_LDAP_3_SERVER_URI: ldap://ipa3.ls.lsst.org + DB_HOST: postgresdb01.ls.lsst.org + LOVE_SITE: base + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 25 + targetCPUUtilizationPercentage: 50 + scaleDownPolicy: + policies: + - type: Pods + value: 2 + periodSeconds: 120 + - type: Percent + value: 10 + periodSeconds: 120 + selectPolicy: Min + resources: + requests: + cpu: 150m + memory: 200Mi + limits: + cpu: 1000m + memory: 1500Mi + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 20 + periodSeconds: 10 + producers: + image: + repository: ts-dockerhub.lsst.org/love-manager + pullPolicy: Always + env: + SERVER_URL: base-lsp.lsst.codes + OLE_API_HOSTNAME: base-lsp.lsst.codes + AUTH_LDAP_1_SERVER_URI: ldap://ipa1.ls.lsst.org + AUTH_LDAP_2_SERVER_URI: ldap://ipa2.ls.lsst.org + AUTH_LDAP_3_SERVER_URI: ldap://ipa3.ls.lsst.org + DB_HOST: postgresdb01.ls.lsst.org + LOVE_SITE: base + autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 25 + targetCPUUtilizationPercentage: 50 + scaleDownPolicy: + policies: + - type: Pods + value: 2 + periodSeconds: 120 + - type: Percent + value: 10 + periodSeconds: 120 + selectPolicy: Min + resources: + requests: + cpu: 150m + memory: 200Mi + limits: + cpu: 1000m + memory: 1500Mi + readinessProbe: + tcpSocket: + port: 8000 + initialDelaySeconds: 20 + periodSeconds: 10 + redis: + image: + repository: redis + tag: '7' + pullPolicy: IfNotPresent + config: | + timeout 60 + viewBackup: + enabled: true + image: + repository: ts-dockerhub.lsst.org/love-view-backup + pullPolicy: Always + schedule: 0 12 * * * + +love-nginx: + image: + repository: nginx + tag: 1.25.1 + pullPolicy: Always + ingress: + hostname: base-lsp.lsst.codes + httpPath: /love + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + initContainers: + frontend: + image: + repository: ts-dockerhub.lsst.org/love-frontend-k8s + pullPolicy: Always + manager: + image: + repository: ts-dockerhub.lsst.org/love-manager-static + pullPolicy: Always + command: + - /bin/sh + - -c + - mkdir -p /usr/src/love-manager; cp -Rv /usr/src/love/manager/media /usr/src/love-manager; cp -Rv /usr/src/love/manager/static /usr/src/love-manager + staticStore: + name: love-nginx-static + storageClass: rook-ceph-block + accessMode: ReadWriteOnce + claimSize: 2Gi + nginxConfig: | + server { + listen 80; + server_name localhost; + location /love { + root /usr/src/love-frontend; + try_files $uri$args $uri$args/ $uri/ /love/index.html; + } + location /love/manager { + client_max_body_size 5M; + proxy_pass http://love-manager-frontend-service:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_redirect off; + } + location /love/manager/producers { + proxy_pass http://love-manager-producers-service:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_redirect off; + } + location /love/media { + alias /usr/src/love-manager/media; + } + location /love/manager/static { + alias /usr/src/love-manager/static; + } + location /love/manager/media { + alias /usr/src/love-manager/media; + } + location /love/simcam { + proxy_pass http://simulation-gencam-service.calsys:5013/; + proxy_set_header Host $host/love; + } + } + loveConfig: | + { + "alarms": { + "minSeveritySound": "mute", + "minSeverityNotification": "mute" + }, + "camFeeds": { + "simcam": "/love/simcam" + }, + "efd": { + "defaultEfdInstance": "base_efd", + "urlStatus": "https://base-lsp.lsst.codes/influxdb/health" + }, + "sal": { + "urlStatus": "https://base-lsp.lsst.codes/sasquatch-rest-proxy/brokers", + "expectedBrokerList": [0, 1, 2] + } + } + +love-producer: + image: + repository: ts-dockerhub.lsst.org/love-producer + pullPolicy: Always + env: + WEBSOCKET_HOST: love-nginx-service/love/manager/producers/ws/subscription + resources: + requests: + cpu: 10m + memory: 100Mi + limits: + cpu: 100m + memory: 300Mi + producers: + - name: ataos + csc: ATAOS:0 + - name: atcamera + csc: ATCamera:0 + - name: atdome + csc: ATDome:0 + - name: atdometrajectory + csc: ATDomeTrajectory:0 + - name: atheaderservice + csc: ATHeaderService:0 + - name: athexapod + csc: ATHexapod:0 + - name: atmcs + csc: ATMCS:0 + - name: atocps + csc: OCPS:1 + - name: atoods + csc: ATOODS:0 + - name: atpneumatics + csc: ATPneumatics:0 + - name: atptg + csc: ATPtg:0 + - name: atscheduler + csc: Scheduler:2 + - name: atscriptqueue + csc: ScriptQueue:2 + - name: atspectrograph + csc: ATSpectrograph:0 + - name: auxteless201 + csc: ESS:201 + - name: auxteless202 + csc: ESS:202 + - name: auxteless203 + csc: ESS:203 + - name: auxteless204 + csc: ESS:204 + - name: auxteless205 + csc: ESS:205 + - name: calibhilless301 + csc: ESS:301 + - name: camerahexapod + csc: MTHexapod:1 + - name: dimm1 + csc: DIMM:1 + - name: dimm2 + csc: DIMM:2 + - name: dsm1 + csc: DSM:1 + - name: dsm2 + csc: DSM:2 + - name: epm1 + csc: EPM:1 + - name: gcheaderservice1 + csc: GCHeaderService:1 + - name: genericcamera1 + csc: GenericCamera:1 + - name: lasertracker1 + csc: LaserTracker:1 + - name: love + csc: LOVE:0 + - name: m2ess106 + csc: ESS:106 + - name: m2hexapod + csc: MTHexapod:2 + - name: mtaircompressor1 + csc: MTAirCompressor:1 + - name: mtaircompressor2 + csc: MTAirCompressor:2 + - name: mtaos + csc: MTAOS:0 + - name: mtdome + csc: MTDome:0 + - name: mtdomeess101 + csc: ESS:101 + - name: mtdomeess102 + csc: ESS:102 + - name: mtdomeess103 + csc: ESS:103 + - name: mtdomeess107 + csc: ESS:107 + - name: mtdomeess108 + csc: ESS:108 + - name: mtdometrajectory + csc: MTDomeTrajectory:0 + - name: mtm1m3 + csc: MTM1M3:0 + - name: mtm2 + csc: MTM2:0 + - name: mtmount + csc: MTMount:0 + - name: mtptg + csc: MTPtg:0 + - name: mtrotator + csc: MTRotator:0 + - name: mtscheduler + csc: Scheduler:1 + - name: mtscriptqueue + csc: ScriptQueue:1 + - name: ocsscheduler + csc: Scheduler:3 + - name: ocsscriptqueue + csc: ScriptQueue:3 + - name: tmaess001 + csc: ESS:1 + - name: tmaess104 + csc: ESS:104 + - name: tmaess105 + csc: ESS:105 + - name: watcher + csc: Watcher:0 + - name: weatherforecast + csc: WeatherForecast:0 From 38cfd09dc7198dfb0110828063da4342e26ab40d Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 12:36:48 -0700 Subject: [PATCH 10/15] BTS: Add obssys configuration. --- applications/obssys/values-base.yaml | 235 +++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 applications/obssys/values-base.yaml diff --git a/applications/obssys/values-base.yaml b/applications/obssys/values-base.yaml new file mode 100644 index 0000000000..ba3cb41bd4 --- /dev/null +++ b/applications/obssys/values-base.yaml @@ -0,0 +1,235 @@ +atqueue: + classifier: scriptqueue2 + image: + repository: ts-dockerhub.lsst.org/scriptqueue + pullPolicy: Always + env: + DAF_BUTLER_REPOSITORY_INDEX: /project/data-repos.yaml + RUN_ARG: 2 --state enabled + USER_USERNAME: user + butlerSecret: + containerPath: &bS-cP /home/saluser/.lsst + dbUser: &bS-dbU oods + secretFilename: &bS-sFn postgres-credentials.txt + nfsMountpoint: + - name: auxtel-gen3-butler + containerPath: /repo/LATISS + readOnly: false + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/repo/LATISS + - name: auxtel-gen3-oods + containerPath: /data/lsstdata/BTS/auxtel + readOnly: true + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/lsstdata/BTS/auxtel + - name: project-shared + containerPath: /project + readOnly: false + server: nfs-project.ls.lsst.org + serverPath: /project + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scriptqueue + topologyKey: "kubernetes.io/hostname" + +atscheduler: + classifier: scheduler2 + image: + repository: ts-dockerhub.lsst.org/scheduler + pullPolicy: Always + env: + INDEX: 2 + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + nfsMountpoint: + - name: rubin-sim-data + containerPath: /home/saluser/rubin_sim_data + readOnly: false + server: nfs-scratch.ls.lsst.org + serverPath: /scratch/scheduler + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scheduler + topologyKey: "kubernetes.io/hostname" + +mtqueue: + classifier: scriptqueue1 + image: + repository: ts-dockerhub.lsst.org/scriptqueue + pullPolicy: Always + env: + DAF_BUTLER_REPOSITORY_INDEX: /project/data-repos.yaml + RUN_ARG: 1 --state enabled + USER_USERNAME: user + butlerSecret: + containerPath: *bS-cP + dbUser: *bS-dbU + secretFilename: *bS-sFn + nfsMountpoint: + - name: auxtel-gen3-butler + containerPath: /repo/LATISS + readOnly: false + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/repo/LATISS + - name: auxtel-gen3-oods + containerPath: /data/lsstdata/BTS/auxtel + readOnly: true + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/lsstdata/BTS/auxtel + - name: project-shared + containerPath: /project + readOnly: false + server: nfs-project.ls.lsst.org + serverPath: /project + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scriptqueue + topologyKey: "kubernetes.io/hostname" + +mtscheduler: + classifier: scheduler1 + image: + repository: ts-dockerhub.lsst.org/scheduler + pullPolicy: Always + env: + INDEX: 1 + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + nfsMountpoint: + - name: rubin-sim-data + containerPath: /home/saluser/rubin_sim_data + readOnly: false + server: nfs-scratch.ls.lsst.org + serverPath: /scratch/scheduler + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scheduler + topologyKey: "kubernetes.io/hostname" + +ocsqueue: + classifier: scriptqueue3 + image: + repository: ts-dockerhub.lsst.org/scriptqueue + pullPolicy: Always + env: + DAF_BUTLER_REPOSITORY_INDEX: /project/data-repos.yaml + RUN_ARG: 3 --state enabled + USER_USERNAME: user + butlerSecret: + containerPath: *bS-cP + dbUser: *bS-dbU + secretFilename: *bS-sFn + nfsMountpoint: + - name: auxtel-gen3-butler + containerPath: /repo/LATISS + readOnly: false + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/repo/LATISS + - name: auxtel-gen3-oods + containerPath: /data/lsstdata/BTS/auxtel + readOnly: true + server: nfs-auxtel.ls.lsst.org + serverPath: /auxtel/lsstdata/BTS/auxtel + - name: project-shared + containerPath: /project + readOnly: false + server: nfs-project.ls.lsst.org + serverPath: /project + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scriptqueue + topologyKey: "kubernetes.io/hostname" + +ocsscheduler: + classifier: scheduler3 + image: + repository: ts-dockerhub.lsst.org/scheduler + pullPolicy: Always + env: + INDEX: 3 + envSecrets: + - name: AWS_ACCESS_KEY_ID + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + key: aws-secret-access-key + - name: MYS3_ACCESS_KEY + key: aws-access-key-id + - name: MYS3_SECRET_KEY + key: aws-secret-access-key + nfsMountpoint: + - name: rubin-sim-data + containerPath: /home/saluser/rubin_sim_data + readOnly: false + server: nfs-scratch.ls.lsst.org + serverPath: /scratch/scheduler + - name: obs-env + containerPath: /net/obs-env + readOnly: true + server: nfs-obsenv.ls.lsst.org + serverPath: /obs-env + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + csc-class: scheduler + topologyKey: "kubernetes.io/hostname" + +watcher: + image: + repository: ts-dockerhub.lsst.org/watcher + pullPolicy: Always From 84ebbb14e703dbf115e8728508415afdf1ddd731 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 14:31:21 -0700 Subject: [PATCH 11/15] BTS: Add simonyitel configuration. --- applications/simonyitel/secrets-base.yaml | 6 ++ applications/simonyitel/secrets.yaml | 2 +- applications/simonyitel/values-base.yaml | 106 ++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 applications/simonyitel/secrets-base.yaml create mode 100644 applications/simonyitel/values-base.yaml diff --git a/applications/simonyitel/secrets-base.yaml b/applications/simonyitel/secrets-base.yaml new file mode 100644 index 0000000000..4defb4bbde --- /dev/null +++ b/applications/simonyitel/secrets-base.yaml @@ -0,0 +1,6 @@ +aws-credentials.ini: + description: >- + S3 Butler credentials in AWS format. + copy: + application: nublado + key: aws-credentials.ini diff --git a/applications/simonyitel/secrets.yaml b/applications/simonyitel/secrets.yaml index c5ad78c684..b404a4ad0d 100644 --- a/applications/simonyitel/secrets.yaml +++ b/applications/simonyitel/secrets.yaml @@ -20,5 +20,5 @@ postgres-credentials.txt: description: >- Credentials for oods user Postgres access. copy: - application: auxtel + application: nublado key: postgres-credentials.txt diff --git a/applications/simonyitel/values-base.yaml b/applications/simonyitel/values-base.yaml new file mode 100644 index 0000000000..4cdc32eb42 --- /dev/null +++ b/applications/simonyitel/values-base.yaml @@ -0,0 +1,106 @@ +lasertracker1-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/lasertracker + pullPolicy: Always + env: + RUN_ARG: 1 --simulate 2 + +mtaircompressor1-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtaircompressor + pullPolicy: Always + env: + RUN_ARG: 1 --simulate --state disabled + +mtaircompressor2-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtaircompressor + pullPolicy: Always + env: + RUN_ARG: 2 --simulate --state disabled + +mtaos: + image: + repository: ts-dockerhub.lsst.org/mtaos + pullPolicy: Always + butlerSecret: + containerPath: /home/saluser/.lsst + dbUser: oods + secretFilename: postgres-credentials.txt + nfsMountpoint: + - name: scratch + containerPath: /scratch + readOnly: false + server: nfs-scratch.tu.lsst.org + serverPath: /scratch + +mtcamhexapod-sim: + enabled: true + classifier: mthexapod1 + image: + repository: ts-dockerhub.lsst.org/mthexapod + pullPolicy: Always + env: + RUN_ARG: --simulate 1 + +mtdome-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtdome + pullPolicy: Always + env: + RUN_ARG: --simulate 1 + +mtdometrajectory: + image: + repository: ts-dockerhub.lsst.org/mtdometrajectory + pullPolicy: Always + +mtm1m3-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtm1m3_sim + pullPolicy: Always + +mtm2-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/m2 + pullPolicy: Always + env: + RUN_ARG: --simulate + +mtm2hexapod-sim: + enabled: true + classifier: mthexapod2 + image: + repository: ts-dockerhub.lsst.org/mthexapod + pullPolicy: Always + env: + RUN_ARG: --simulate 2 + +mtmount-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtmount + pullPolicy: Always + env: + RUN_ARG: --simulate + +mtptg: + image: + repository: ts-dockerhub.lsst.org/ptkernel + pullPolicy: Always + env: + TELESCOPE: MT + +mtrotator-sim: + enabled: true + image: + repository: ts-dockerhub.lsst.org/mtrotator + pullPolicy: Always + env: + RUN_ARG: --simulate From 74dcc3d1a8966f551c16c7f9b084f15f42f601cd Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 16:07:27 -0700 Subject: [PATCH 12/15] BTS: Activate telescope apps. --- environments/values-base.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/environments/values-base.yaml b/environments/values-base.yaml index 4369a1eb04..5b6c304c7a 100644 --- a/environments/values-base.yaml +++ b/environments/values-base.yaml @@ -5,18 +5,26 @@ vaultPathPrefix: "secret/k8s_operator/base-lsp.lsst.codes" applications: argo-workflows: true + auxtel: true + calsys: true consdb: true + control-system-test: true + eas: true exposurelog: true + love: true narrativelog: true nightreport: true nublado: true + obssys: true portal: true rubintv: true sasquatch: true + simonyitel: true squareone: true strimzi: true telegraf: true telegraf-ds: true + uws: true controlSystem: imageTag: "k0001" From 3f4d8ab49595e06c89246a42fb895f97334d8097 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Tue, 2 Jul 2024 16:08:45 -0700 Subject: [PATCH 13/15] BTS: Add labels to sasquatch namespace. --- environments/values-base.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environments/values-base.yaml b/environments/values-base.yaml index 5b6c304c7a..565b8eca54 100644 --- a/environments/values-base.yaml +++ b/environments/values-base.yaml @@ -2,6 +2,9 @@ name: "base" fqdn: "base-lsp.lsst.codes" appOfAppsName: "science-platform" vaultPathPrefix: "secret/k8s_operator/base-lsp.lsst.codes" +namespaceLabels: + sasquatch: + lsst.io/discover: 'true' applications: argo-workflows: true From e1e4452f4b57fb936a7a8297f5b0e5782d7e3d06 Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Fri, 5 Jul 2024 15:16:19 -0700 Subject: [PATCH 14/15] BTS: More secrets for simonyitel app. --- applications/simonyitel/secrets-base.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/applications/simonyitel/secrets-base.yaml b/applications/simonyitel/secrets-base.yaml index 4defb4bbde..8d237221db 100644 --- a/applications/simonyitel/secrets-base.yaml +++ b/applications/simonyitel/secrets-base.yaml @@ -1,3 +1,12 @@ +oods-latiss-secret-key: + description: >- + S3 secret key for ATOODS oods-latiss user. +oods-lsstcam-secret-key: + description: >- + S3 secret key for MTOODS oods-lsstcam user. +butler-secret-key: + description: >- + S3 secret key for butler user. aws-credentials.ini: description: >- S3 Butler credentials in AWS format. From 7c0ac6bcbea9a76c8becbbe206c782fa3ebb54af Mon Sep 17 00:00:00 2001 From: Michael Reuter Date: Mon, 8 Jul 2024 14:23:35 -0700 Subject: [PATCH 15/15] Fix linter. --- applications/uws/values-base.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/uws/values-base.yaml b/applications/uws/values-base.yaml index 90eaa67877..c9ef439fe2 100644 --- a/applications/uws/values-base.yaml +++ b/applications/uws/values-base.yaml @@ -53,4 +53,3 @@ atocps: pullPolicy: Always env: RUN_ARG: 1 -