diff --git a/images/taginfo/start.sh b/images/taginfo/start.sh index 2733a50e..951d6a41 100755 --- a/images/taginfo/start.sh +++ b/images/taginfo/start.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -x WORKDIR=/usr/src/app DATADIR=/usr/src/app/data DATADOWNLOAD=/osm/planet/var @@ -39,14 +40,14 @@ process_data() { db/update.sh $DATADIR master/update.sh $DATADIR projects/update.sh $DATADIR - # # languages/update.sh $DATADIR - # # wiki/update.sh $DATADIR - # # wikidata/update.sh $DATADIR + cp $DATADIR/selection.db $DATADIR/../ + # languages/update.sh $DATADIR + # wiki/update.sh $DATADIR + # wikidata/update.sh $DATADIR chronology/update.sh $DATADIR ./update_all.sh $DATADIR mv $DATADIR/*.db $DATADIR/ mv $DATADIR/*/*.db $DATADIR/ - # if BUCKET_NAME is set upload data if ! aws s3 ls "s3://$BUCKET_NAME/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then aws s3 sync $DATADIR/ s3://$AWS_S3_BUCKET/$ENVIRONMENT/ --exclude "*" --include "*.db" @@ -64,6 +65,8 @@ compress_files() { download_db_files() { if ! aws s3 ls "s3://$AWS_S3_BUCKET/$ENVIRONMENT" 2>&1 | grep -q 'An error occurred'; then aws s3 sync "s3://$AWS_S3_BUCKET/$ENVIRONMENT/" "$DATADIR/" + mv $DATADIR/*.db $DATADIR/ + mv $DATADIR/*/*.db $DATADIR/ compress_files fi } diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile index 10b7fd2b..e99ccc28 100644 --- a/images/tiler-imposm/Dockerfile +++ b/images/tiler-imposm/Dockerfile @@ -41,6 +41,8 @@ ENV PATH $PATH:/usr/local/go/bin WORKDIR /go ENV GOPATH /go RUN go get github.com/omniscale/imposm3 +WORKDIR $GOPATH/src/github.com/omniscale/imposm3 +RUN git checkout v0.11.1 RUN go install github.com/omniscale/imposm3/cmd/imposm ENV PATH $PATH:$GOPATH/bin diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh index c9fa48a0..dfffdc0c 100755 --- a/images/tiler-imposm/start.sh +++ b/images/tiler-imposm/start.sh @@ -1,9 +1,5 @@ #!/bin/bash -set -e - -STATEFILE="state.txt" -PBFFILE="osm.pbf" -LIMITFILE="limitFile.geojson" +set -ex # directories to keep the imposm's cache for updating the db WORKDIR=/mnt/data @@ -11,6 +7,9 @@ CACHE_DIR=$WORKDIR/cachedir DIFF_DIR=$WORKDIR/diff IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir +PBFFILE="${WORKDIR}/osm.pbf" +STATEFILE="state.txt" +LIMITFILE="limitFile.geojson" # # Setting directory # settingDir=/osm # Folder to store the imposm expider files in s3 or gs @@ -33,18 +32,9 @@ mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR" function getData() { ### Get the PBF file from the cloud provider or public URL if [ "$TILER_IMPORT_FROM" == "osm" ]; then - wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE" - elif [ "$TILER_IMPORT_FROM" == "osmseed" ]; then - if [ "$CLOUDPROVIDER" == "aws" ]; then - # Get the state.txt file from S3 - aws s3 cp "$AWS_S3_BUCKET/planet/full-history/$STATEFILE" . - PBFCloudPath=$(tail -n +1 "$STATEFILE") - aws s3 cp "$PBFCloudPath" "$PBFFILE" - elif [ "$CLOUDPROVIDER" == "gcp" ]; then - # Get the state.txt file from GS - gsutil cp "$GCP_STORAGE_BUCKET/planet/full-history/$STATEFILE" . - PBFCloudPath=$(tail -n +1 "$STATEFILE") - gsutil cp "$PBFCloudPath" "$PBFFILE" + if [ ! -f "$PBFFILE" ]; then + echo "$PBFFILE does not exist, downloading..." + wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE" fi fi } @@ -123,26 +113,30 @@ function importData() { echo "Execute the missing functions" psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql - echo "Import Natural Earth..." - ./scripts/natural_earth.sh - - echo "Import OSM Land..." - ./scripts/osm_land.sh + if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then + echo "Importing Natural Earth..." + ./scripts/natural_earth.sh + fi + if [ "$IMPORT_OSM_LAND" = "true" ]; then + echo "Import OSM Land..." + ./scripts/osm_land.sh + fi + echo "Import PBF file..." if [ -z "$TILER_IMPORT_LIMIT" ]; then imposm import \ -config $WORKDIR/config.json \ -read $PBFFILE \ -write \ - -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR + -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR else wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE imposm import \ -config $WORKDIR/config.json \ -read $PBFFILE \ -write \ - -diff -cachedir $CACHE_DIR -diffdir $DIFF_DIR \ + -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR \ -limitto $WORKDIR/$LIMITFILE fi diff --git a/osm-seed/templates/db/db-statefulset.yaml b/osm-seed/templates/db/db-statefulset.yaml index a8fc04a0..6c5afe0c 100644 --- a/osm-seed/templates/db/db-statefulset.yaml +++ b/osm-seed/templates/db/db-statefulset.yaml @@ -43,6 +43,8 @@ spec: value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }} - name: PGUSER value: {{ default "postgres" .Values.db.env.POSTGRES_USER | quote }} + - name: ENVIRONMENT + value: {{ .Values.environment }} - name: PGDATA value: {{ .Values.db.persistenceDisk.mountPath }} - name: POD_IP diff --git a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml index ab344bab..dbe39040 100644 --- a/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml +++ b/osm-seed/templates/tiler-imposm/tiler-imposm-statefulset.yaml @@ -66,6 +66,10 @@ spec: value: {{ .Values.tilerImposm.env.OVERWRITE_STATE | quote }} - name: UPLOAD_EXPIRED_FILES value: {{ .Values.tilerImposm.env.UPLOAD_EXPIRED_FILES | default "true" | quote }} + - name: IMPORT_NATURAL_EARTH + value: {{ .Values.tilerImposm.env.IMPORT_NATURAL_EARTH | default "true" | quote }} + - name: IMPORT_OSM_LAND + value: {{ .Values.tilerImposm.env.IMPORT_OSM_LAND | default "true" | quote }} - name: CLOUDPROVIDER value: {{ .Values.cloudProvider }} # In case cloudProvider=aws diff --git a/osm-seed/templates/web/web-deployment.yaml b/osm-seed/templates/web/web-deployment.yaml index dc7bbb03..b62eb063 100644 --- a/osm-seed/templates/web/web-deployment.yaml +++ b/osm-seed/templates/web/web-deployment.yaml @@ -125,6 +125,8 @@ spec: value: {{ .Values.web.env.NEW_RELIC_APP_NAME | default ""}} - name: ORGANIZATION_NAME value: {{ .Values.web.env.ORGANIZATION_NAME | default "OpenStreetMap"}} + - name: WEBSITE_STATUS + value: {{ .Values.web.env.WEBSITE_STATUS | default "online"}} volumeMounts: - mountPath: /dev/shm name: shared-memory diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index b1500840..1ea57294 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -73,7 +73,7 @@ db: env: POSTGRES_DB: openstreetmap POSTGRES_USER: postgres - POSTGRES_PASSWORD: "1234" + POSTGRES_PASSWORD: 1234 persistenceDisk: enabled: false mountPath: /var/lib/postgresql/data @@ -125,6 +125,7 @@ web: OAUTH_CLIENT_ID: "abcd" # Create client ID and key as OAuth2 Application OAUTH_KEY: "xyz" ORGANIZATION_NAME: "OSMSeed" + WEBSITE_STATUS: "online" resources: enabled: false requests: @@ -377,6 +378,8 @@ tilerImposm: TILER_IMPORT_PBF_URL: http://download.geofabrik.de/europe/monaco-latest.osm.pbf TILER_IMPORT_LIMIT: https://gist.githubusercontent.com/Rub21/96bdcac5eb11f0b36ba8d0352ac537f4/raw/2606f2e207d4a0d895897a83efa1efacefd36eb4/monaco.geojson UPLOAD_EXPIRED_FILES: true + IMPORT_NATURAL_EARTH: true + IMPORT_OSM_LAND: true persistenceDisk: enabled: false accessMode: ReadWriteOnce