Merge pull request #1 from MOV-AI/dev

Initial commit

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/autoupdate.yml

@@ -0,0 +1,15 @@
+# autoupdate is a GitHub Action that auto-updates pull requests branches whenever changes land on their destination branch.
+name: autoupdate
+on:
+  push:
+    branches:
+      - main
+      - dev
+jobs:
+  autoupdate:
+    name: autoupdate
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: docker://chinthakagodawita/autoupdate-action:v1
+        env:
+          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'

.github/workflows/bump-version.yml

@@ -0,0 +1,19 @@
+name: Bump version
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: '0'
+    - name: Bump version and push tag
+      uses: anothrNick/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        WITH_V: true
+        DEFAULT_BUMP: patch
+        RELEASE_BRANCHES: main

.github/workflows/docker-ci.yml

@@ -0,0 +1,26 @@
+name: "Build&Deploy redis images"
+
+on:
+  # Trigger the workflow on push or pull request
+  # but only for the main branch
+  # or on tag of type v*.*.*
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - main
+  release:
+    types: [released]
+jobs:
+  redis:
+    uses: MOV-AI/.github/.github/workflows/docker-workflow.yml@main
+    with:
+      docker_file: Dockerfile
+      docker_image: registry.cloud.mov.ai/devops/redis
+      github_ref: ${{ github.ref }}
+    secrets:
+      registry_user: ${{ secrets.PORTUS_APP_USER }}
+      registry_password: ${{ secrets.PORTUS_APP_TOKEN }}

Dockerfile

@@ -0,0 +1,22 @@
+# This Dockerfile is the base image for Mov.ai Redis
+FROM redislabs/rejson:2.0.7
+# Labels
+LABEL description="MOV.AI Redis Image"
+LABEL maintainer="[email protected]"
+LABEL movai="redis"
+
+# Configure and install
+COPY files/etc/ /etc/
+COPY files/bin/ /usr/local/bin/
+
+RUN apt-get update &&\
+    apt-get -y install --no-install-recommends redis-tools \
+    build-essential apt-transport-https curl python3 python3-dev python3-pip python3-setuptools software-properties-common unzip wget gnupg &&\
+    apt-get clean -y > /dev/null &&\
+    /usr/bin/pip3 install wheel rdbtools python-lzf &&\
+    rm -rf /var/cache/apt/* &&\
+    rm -rf /var/lib/apt/lists/*  &&\
+    rm -rf /tmp/* &&\
+    mkdir -p /default
+
+ENTRYPOINT ["movai-entrypoint.sh","/etc/redis.conf"]

files/bin/dump_database.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Copyright 2021 MOV.AI
+#
+#    Licensed under the Mov.AI License version 1.0;
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        https://www.mov.ai/flow-license/
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# File: install-packages.sh
+# File: dump_database.sh
+set -eo pipefail
+
+function dump_all() {
+    mkdir -p ${TEMP_DIR}/all
+    rdb --command protocol ${RDBFILE} --file ${TEMP_DIR}/all/all.data
+    cat ${TEMP_DIR}/all/*.data | gzip > ${OUTPUT_DIR}/all_db_patch.gz
+}
+
+function dump_all_core() {
+    mkdir -p ${TEMP_DIR}/core
+    rdb --command protocol --key "Ports:" ${RDBFILE} --file ${TEMP_DIR}/core/ports.data
+    rdb --command protocol --key "System:" ${RDBFILE} --file ${TEMP_DIR}/core/system.data
+    rdb --command protocol --key "Callback:server4" ${RDBFILE} --file ${TEMP_DIR}/core/backend_callback.data
+    rdb --command protocol --key "Node:server4" ${RDBFILE} --file ${TEMP_DIR}/core/backend_node.data
+    cat ${TEMP_DIR}/core/*.data | gzip > ${OUTPUT_DIR}/core_db_patch.gz
+}
+
+function dump_all_runtime () {
+    mkdir -p ${TEMP_DIR}/runtime
+    rdb --command protocol --key "Flow:" ${RDBFILE} --file ${TEMP_DIR}/runtime/flow.data
+    rdb --command protocol --key "Layout:" ${RDBFILE} --file ${TEMP_DIR}/runtime/layout.data
+    rdb --command protocol --key "GraphicScene:" ${RDBFILE} --file ${TEMP_DIR}/runtime/graph_scene.data
+    rdb --command protocol --key "GraphicAsset:" ${RDBFILE} --file ${TEMP_DIR}/runtime/graph_asset.data
+    rdb --command protocol --key "Form:" ${RDBFILE} --file ${TEMP_DIR}/runtime/form.data
+    rdb --command protocol --key "Annotation:" ${RDBFILE} --file ${TEMP_DIR}/runtime/annotation.data
+    rdb --command protocol --key "StateMachine:" ${RDBFILE} --file ${TEMP_DIR}/runtime/state_machine.data
+    cat ${TEMP_DIR}/runtime/*.data | gzip > ${OUTPUT_DIR}/runtime_db_patch.gz
+}
+
+function dump_all_libraries () {
+    mkdir -p ${TEMP_DIR}/libraries
+    rdb --command protocol --key "Callback:" -o "Callback:server4" ${RDBFILE} --file ${TEMP_DIR}/libraries/callback.data
+    rdb --command protocol --key "Node:" -o "Node:server4" ${RDBFILE} --file ${TEMP_DIR}/libraries/node.data
+    rdb --command protocol --key "Message:" ${RDBFILE} --file ${TEMP_DIR}/libraries/message.data
+    cat ${TEMP_DIR}/libraries/*.data | gzip > ${OUTPUT_DIR}/libraries_db_patch.gz
+}
+
+RDBFILE=/data/dump.rdb
+TIMESTAMP=$(date "+%Y%m%d-%H%M%S")
+OUTPUT_DIR=/data/exports/dump-${TIMESTAMP}
+
+for ARG in "${@:1}"
+do
+    case ${ARG} in
+        all*)
+            TEMP_DIR=$(mktemp -d)
+            # make sure we have the latest data on disk
+            redis-cli SAVE > /dev/null
+            mkdir -p ${OUTPUT_DIR}
+            dump_all
+            cd ${OUTPUT_DIR}
+            tar czf update.tar.gz all_db_patch.gz
+            rm -rf ${TEMP_DIR}
+            echo ${OUTPUT_DIR}
+            exit 0
+            ;;
+        core*)
+            TEMP_DIR=$(mktemp -d)
+            # make sure we have the latest data on disk
+            redis-cli SAVE > /dev/null
+            mkdir -p ${OUTPUT_DIR}
+            dump_all_core
+            cd ${OUTPUT_DIR}
+            tar czf update.tar.gz core_db_patch.gz
+            rm -rf ${TEMP_DIR}
+            echo ${OUTPUT_DIR}
+            exit 0
+            ;;
+        runtime)
+            TEMP_DIR=$(mktemp -d)
+            # make sure we have the latest data on disk
+            redis-cli SAVE > /dev/null
+            mkdir -p ${OUTPUT_DIR}
+            dump_all_runtime
+            cd ${OUTPUT_DIR}
+            tar czf update.tar.gz runtime_db_patch.gz
+            rm -rf ${TEMP_DIR}
+            exit 0
+            ;;
+        libraries*)
+            TEMP_DIR=$(mktemp -d)
+            # make sure we have the latest data on disk
+            redis-cli SAVE > /dev/null
+            mkdir -p ${OUTPUT_DIR}
+            dump_all_libraries
+            cd ${OUTPUT_DIR}
+            tar czf update.tar.gz libraries_db_patch.gz
+            rm -rf ${TEMP_DIR}
+            echo ${OUTPUT_DIR}
+            exit 0
+            ;;
+    esac
+done
+
+exit 1

files/bin/health_check.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright 2021 MOV.AI
+#
+#    Licensed under the Mov.AI License version 1.0;
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        https://www.mov.ai/flow-license/
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# File: health_check.sh
+set -eo pipefail
+
+host="127.0.0.1"
+
+if ping="$(redis-cli -h "$host" ping)" && [ "$ping" = 'PONG' ]; then
+	exit 0
+fi
+
+exit 1

files/bin/initial_setup.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright 2021 MOV.AI
+#
+#    Licensed under the Mov.AI License version 1.0;
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        https://www.mov.ai/flow-license/
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# File: initial_setup.sh
+set -eo pipefail
+
+if [ -f /data/.initial_db_import_done ]; then
+	printf "Database already initialized, ignoring....\n"
+	exit 0
+fi
+
+printf "Initializing database\n"
+apt install /default/movai-*.deb

files/bin/movai-entrypoint.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Copyright 2021 MOV.AI
+#
+#    Licensed under the Mov.AI License version 1.0;
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        https://www.mov.ai/flow-license/
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# File: docker-entrypoint.sh
+set -m
+
+# Run command in background
+exec docker-entrypoint.sh ${@} &
+
+HOST=localhost
+PORT=6379
+
+printf "Waiting redis on %s:.\n" "$HOST:$PORT"
+while ! redis-cli -h $HOST -p $PORT ping | grep -q PONG; do
+    sleep 1
+    printf "."
+done
+printf "\nRedis is UP\n"
+
+# Run any needed APT install
+if [ -n "${APT_AUTOINSTALL}" ]; then
+    if [ -f "${MOVAI_HOME}/.first_run_autoinstall" ] && [ "${APT_AUTOINSTALL}" = "once"  ]; then
+        printf "APT autoinstall: skipped\n"
+    else
+        printf "APT autoinstall:\n"
+        # If we have apt keys to add
+        if [ -n "${APT_KEYS_URL_LIST}" ]; then
+            for key_url in ${APT_KEYS_URL_LIST//,/ }; do
+                printf "APT Key add: %s\n" "${key_url}"
+                curl -fsSL "${key_url}" | apt-key add -
+            done
+        fi
+
+        # Switching separator to comma
+        SAVEIFS=$IFS
+        IFS=,
+
+        # If we have apt repos to add
+        if [ -n "${APT_REPOS_LIST}" ]; then
+            for ppa in ${APT_REPOS_LIST}; do
+                printf "APT Repo add: %s\n" "${ppa}"
+                if add-apt-repository -y "${ppa}" > /dev/null 2>&1; then
+                    printf "OK\n"
+                else
+                    printf "FAILED\n"
+                fi
+            done
+        fi
+
+        # If we have packages on our env var we do install
+        if [ -n "${APT_INSTALL_LIST}" ]; then
+            printf "APT Install list: %s\n" "${APT_INSTALL_LIST}"
+            apt-get update
+            DEBIAN_FRONTEND=noninteractive apt-get --quiet -y --no-install-recommends install ${APT_INSTALL_LIST}
+            apt-get clean -y
+        fi
+
+        # Switching back separator to default
+        IFS=$SAVEIFS
+        touch "${MOVAI_HOME}/.first_run_autoinstall"
+        printf "APT autoinstall: done\n"
+    fi
+fi
+
+# now we bring the primary process back into the foreground
+fg %
+
+# remove job control
+set +m

files/bin/sync.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Start wminput in the background and send its output to file descriptor 3
+RET=0
+exec 3< <(redis-cli sync 2>&1)
+PID=$!
+# Read the output of wminput line by line until one line contains Ready
+while read line; do
+   case "$line" in
+   *failed*)
+      kill $PID
+      RET=1
+      break
+      ;;
+   *SYNC\ done.*)
+      kill $PID
+      break
+      ;;
+   *)
+      ;;
+   esac
+done <&3
+# Close the file descriptor
+exec 3<&-
+
+exit $RET