Pass API database configuration through environment variables (#322)

* Pass API database configuration through environment variables * Update quote for env vars * Add website status in the env var * Remove env vars for postgres * Add env var ENVIRONMENT for api-db * Fix to get ENVIRONMENT * Update imposm import script * Move taginfo db files to data dir * Update taginfo script (#325) * Move db files for taginfo * Update taginfo script
developmentseed · May 2, 2024 · 7c38231 · 7c38231
1 parent ec665eb
commit 7c38231
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 29 deletions.
diff --git a/images/taginfo/start.sh b/images/taginfo/start.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -x
 WORKDIR=/usr/src/app
 DATADIR=/usr/src/app/data
 DATADOWNLOAD=/osm/planet/var
@@ -39,14 +40,14 @@ process_data() {
     db/update.sh $DATADIR
     master/update.sh $DATADIR
     projects/update.sh $DATADIR
-    # # languages/update.sh $DATADIR
-    # # wiki/update.sh $DATADIR
-    # # wikidata/update.sh $DATADIR
+    cp $DATADIR/selection.db $DATADIR/../
+    # languages/update.sh $DATADIR
+    # wiki/update.sh $DATADIR
+    # wikidata/update.sh $DATADIR
     chronology/update.sh $DATADIR
     ./update_all.sh $DATADIR
     mv $DATADIR/*.db $DATADIR/
     mv $DATADIR/*/*.db $DATADIR/
-
     # if BUCKET_NAME is set upload data
     if ! aws s3 ls "s3://$BUCKET_NAME/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then
         aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/$ENVIRONMENT/  --exclude "*" --include "*.db"
@@ -64,6 +65,8 @@ compress_files() {
 download_db_files() {
     if ! aws s3 ls "s3://$AWS_S3_BUCKET/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then
         aws s3 sync "s3://$AWS_S3_BUCKET/$ENVIRONMENT/" "$DATADIR/"
+        mv $DATADIR/*.db $DATADIR/
+        mv $DATADIR/*/*.db $DATADIR/
         compress_files
     fi
 }

diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile
@@ -41,6 +41,8 @@ ENV PATH $PATH:/usr/local/go/bin
 WORKDIR /go
 ENV GOPATH /go
 RUN go get github.com/omniscale/imposm3
+WORKDIR $GOPATH/src/github.com/omniscale/imposm3
+RUN git checkout v0.11.1
 RUN go install github.com/omniscale/imposm3/cmd/imposm
 ENV PATH $PATH:$GOPATH/bin
 

diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -1,16 +1,15 @@
 #!/bin/bash
-set -e
-
-STATEFILE="state.txt"
-PBFFILE="osm.pbf"
-LIMITFILE="limitFile.geojson"
+set -ex
 
 # directories to keep the imposm's cache for updating the db
 WORKDIR=/mnt/data
 CACHE_DIR=$WORKDIR/cachedir
 DIFF_DIR=$WORKDIR/diff
 IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir
 
+PBFFILE="${WORKDIR}/osm.pbf"
+STATEFILE="state.txt"
+LIMITFILE="limitFile.geojson"
 # # Setting directory
 # settingDir=/osm
 # Folder to store the imposm expider files in s3 or gs
@@ -33,18 +32,9 @@ mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR"
 function getData() {
     ### Get the PBF file from the cloud provider or public URL
     if [ "$TILER_IMPORT_FROM" == "osm" ]; then
-        wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
-    elif [ "$TILER_IMPORT_FROM" == "osmseed" ]; then
-        if [ "$CLOUDPROVIDER" == "aws" ]; then
-            # Get the state.txt file from S3
-            aws s3 cp "$AWS_S3_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            aws s3 cp "$PBFCloudPath" "$PBFFILE"
-        elif [ "$CLOUDPROVIDER" == "gcp" ]; then
-            # Get the state.txt file from GS
-            gsutil cp "$GCP_STORAGE_BUCKET/planet/full-history/$STATEFILE" .
-            PBFCloudPath=$(tail -n +1 "$STATEFILE")
-            gsutil cp "$PBFCloudPath" "$PBFFILE"
+        if [ ! -f "$PBFFILE" ]; then
+          echo "$PBFFILE does not exist, downloading..."
+          wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE"
         fi
     fi
 }
@@ -123,26 +113,30 @@ function importData() {
     echo "Execute the missing functions"
     psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql
 
-    echo "Import Natural Earth..."
-    ./scripts/natural_earth.sh
-
-    echo "Import OSM Land..."
-    ./scripts/osm_land.sh
+    if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then
+        echo "Importing Natural Earth..."
+        ./scripts/natural_earth.sh
+    fi
 
+    if [ "$IMPORT_OSM_LAND" = "true" ]; then
+        echo "Import OSM Land..."
+        ./scripts/osm_land.sh
+    fi
+
     echo "Import PBF file..."
     if [ -z "$TILER_IMPORT_LIMIT" ]; then
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR
     else
         wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE
         imposm import \
             -config $WORKDIR/config.json \
             -read $PBFFILE \
             -write \
-            -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR \
+            -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR \
             -limitto $WORKDIR/$LIMITFILE
     fi
 

diff --git a/osm-seed/templates/db/db-statefulset.yaml b/osm-seed/templates/db/db-statefulset.yaml
@@ -43,6 +43,8 @@ spec:
               value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }}
             - name: PGUSER
               value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }}
+            - name: ENVIRONMENT
+              value: {{ .Values.environment }}
             - name: PGDATA
               value: {{ .Values.db.persistenceDisk.mountPath }}
             - name: POD_IP

diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml
@@ -66,6 +66,10 @@ spec:
             value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }}
           - name: UPLOAD_EXPIRED_FILES
             value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | default "true" | quote }}
+          - name: IMPORT_NATURAL_EARTH
+            value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }}
+          - name: IMPORT_OSM_LAND
+            value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }}
           - name: CLOUDPROVIDER
             value: {{ .Values.cloudProvider }}
           # In case cloudProvider=aws

diff --git a/osm-seed/templates/web/web-deployment.yaml b/osm-seed/templates/web/web-deployment.yaml
@@ -125,6 +125,8 @@ spec:
               value: {{ .Values.web.env.NEW_RELIC_APP_NAME | default ""}}
             - name: ORGANIZATION_NAME
               value: {{ .Values.web.env.ORGANIZATION_NAME | default "OpenStreetMap"}}
+            - name: WEBSITE_STATUS
+              value: {{ .Values.web.env.WEBSITE_STATUS | default "online"}}
           volumeMounts:
             - mountPath: /dev/shm
               name: shared-memory

diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml
@@ -73,7 +73,7 @@ db:
   env:
     POSTGRES_DB: openstreetmap
     POSTGRES_USER: postgres
-    POSTGRES_PASSWORD: "1234"
+    POSTGRES_PASSWORD: 1234
   persistenceDisk:
     enabled: false
     mountPath: /var/lib/postgresql/data
@@ -125,6 +125,7 @@ web:
     OAUTH_CLIENT_ID: "abcd" # Create client ID and key as OAuth2 Application
     OAUTH_KEY: "xyz"
     ORGANIZATION_NAME: "OSMSeed"
+    WEBSITE_STATUS: "online"
   resources:
     enabled: false
     requests:
@@ -377,6 +378,8 @@ tilerImposm:
     TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf
     TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson
     UPLOAD_EXPIRED_FILES: true
+    IMPORT_NATURAL_EARTH: true
+    IMPORT_OSM_LAND: true
   persistenceDisk:
     enabled: false
     accessMode: ReadWriteOnce