Updates for memsql 8 1

.circleci/config.yml

@@ -7,6 +7,8 @@ commands:
             - run: |
                   sudo apt update
                   sudo apt install -y make bash git jq docker
+                  sudo python3 -m pip install --upgrade pip
+                  sudo pip3 install boto3
 
 jobs:
     test:
@@ -19,26 +21,30 @@ jobs:
             - run: make test
             - run: make test-destroy
 
-    redhat:
+    dockerhub:
         parameters:
             package:
                 type: string
-        machine: true
-        steps:
-            - setup_environment
-            - checkout
-            - run: docker login -u unused --password ${REDHAT_REGISTRY_KEY_NODE} scan.connect.redhat.com
-            - run: make build-<<parameters.package>> redhat-verify-<<parameters.package>> VARIANT=redhat
-
-    redhat-ciab:
-        machine: true
+        machine:
+            # https://circleci.com/docs/2.0/images/linux-vm/16.04-to-20.04-migration/
+            image: ubuntu-2004:202201-02
         steps:
             - setup_environment
             - checkout
-            - run: docker login -u unused --password ${REDHAT_REGISTRY_KEY_CIAB} scan.connect.redhat.com
-            - run: make build-ciab redhat-verify-ciab VARIANT=redhat
+            - run: docker login -u ${DOCKER_HUB_USER} --password ${DOCKER_HUB_KEY}
+            - run: make build-<<parameters.package>> publish-<<parameters.package>>
+            - when:
+                condition:
+                    matches:
+                        pattern: "^node(-.+)?$"
+                        value: <<parameters.package>>
+                steps:
+                    - run: docker logout
+                    - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json
+                    - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io
+                    - run: make publish-gcr-<<parameters.package>>
 
-    dockerhub:
+    gcr-public:
         parameters:
             package:
                 type: string
@@ -48,7 +54,8 @@ jobs:
         steps:
             - setup_environment
             - checkout
-            - run: docker login -u ${DOCKER_HUB_USER} --password ${DOCKER_HUB_KEY}
+            - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json
+            - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io
             - run: make build-<<parameters.package>> publish-<<parameters.package>>
 
     ubi-gcr-internal:
@@ -58,7 +65,7 @@ jobs:
             - checkout
             # We can't use Bash variables in CircleCI so we just duplicate the ""./dockerconfig.json"
             # string here.
-            - run: echo "${GCLOUD_SERVICE_ACCOUNT_INTERNAL}" | base64 -d >./dockerconfig.json
+            - run: echo "${GCLOUD_SERVICE_ACCOUNT_SINGLESTORE_PUBLIC}" | base64 -d >./dockerconfig.json
             - run: cat ./dockerconfig.json | docker login -u _json_key --password-stdin https://gcr.io
             - run: make build-node redhat-verify-ubi-gcr-internal-node VARIANT=redhat
 
@@ -83,36 +90,16 @@ workflows:
                                   "node-7-3",
                                   "node-7-5",
                                   "node-7-6",
+                                  "node-7-8",
+                                  "node-8-0",
+                                  "node-8-1",
                                   "dynamic-node",
                                   "ciab",
                                   "tools",
                               ]
                   requires:
                       - test
 
-            - publish-redhat:
-                  name: publish-redhat-<< matrix.package >>
-                  type: approval
-                  matrix:
-                      parameters:
-                          # Must match line 93.
-                          package:
-                              [
-                                  "node",
-                                  "node-7-0",
-                                  "node-7-1",
-                                  "node-7-3",
-                                  "node-7-5",
-                                  "node-7-6"
-                              ]
-                  requires:
-                      - test
-
-            - publish-redhat-ciab:
-                  name: publish-redhat-ciab
-                  type: approval
-                  requires:
-                      - test
 
             - dockerhub:
                   name: dockerhub-<< matrix.package >>
@@ -128,34 +115,16 @@ workflows:
                                   "node-7-3",
                                   "node-7-5",
                                   "node-7-6",
+                                  "node-7-8",
+                                  "node-8-0",
+                                  "node-8-1",
                                   "dynamic-node",
                                   "ciab",
                                   "tools",
                               ]
                   requires:
                       - publish-dockerhub-<< matrix.package >>
 
-            - redhat:
-                  name: redhat-<< matrix.package >>
-                  matrix:
-                      parameters:
-                          package:
-                              [
-                                  "node",
-                                  "node-7-0",
-                                  "node-7-1",
-                                  "node-7-3",
-                                  "node-7-5",
-                                  "node-7-6"
-                              ]
-                  requires:
-                      - publish-redhat-<< matrix.package >>
-
-            - redhat-ciab:
-                  name: redhat-ciab
-                  requires:
-                      - publish-redhat-ciab
-
             - publish-ubi-gcr-internal:
                   name: publish-ubi-gcr-internal
                   type: approval
@@ -165,4 +134,28 @@ workflows:
             - ubi-gcr-internal:
                   name: ubi-gcr-internal
                   requires:
-                      - publish-ubi-gcr-internal
+                      - publish-ubi-gcr-internal
+
+            - publish-gcr-public:
+                  name: publish-gcr-public-<< matrix.package >>
+                  type: approval
+                  matrix:
+                      parameters:
+                          # Must match line 71.
+                          package:
+                              [
+                                  "node-cloud",
+                              ]
+                  requires:
+                      - test
+
+            - gcr-public:
+                name: gcr-public-<< matrix.package >>
+                matrix:
+                    parameters:
+                        package:
+                            [
+                                "node-cloud",
+                            ]
+                requires:
+                    - publish-gcr-public-<< matrix.package >>

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.vscode
+lib

Dockerfile-base

@@ -16,10 +16,12 @@ RUN yum install yum-utils -y
 RUN yum-config-manager --save --setopt=skip_missing_names_on_install=0
 
 # updates and basic utilities
-ARG RELEASE_CHANNEL=production
+ARG RELEASE_CHANNEL
+RUN rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux
+
 # currently yum update-minimal fails on curl, so we need to update curl before
 RUN yum update curl -y && yum -y update-minimal --setopt=tsflags=nodocs \
-      --security --sec-severity=Important --sec-severity=Critical \
+      --security --sec-severity=Important --sec-severity=Critical --nobest \
  && yum install -y yum-utils wget procps
 
 RUN yum-config-manager --add-repo https://release.memsql.com/${RELEASE_CHANNEL}/rpm/x86_64/repodata/memsql.repo \
@@ -28,17 +30,8 @@ RUN yum-config-manager --add-repo https://release.memsql.com/${RELEASE_CHANNEL}/
 RUN dnf --enablerepo=* clean all && dnf update -y; exit 0
 
 # address vulnarabilities found by `grype --only-fixed` command
-RUN yum remove vim-minimal platform-python-pip.noarch -y && yum update expat libxml2 libgcrypt -y
-
-# jq 1.6
-ENV JQ_VERSION='1.6'
-RUN wget --no-check-certificate https://raw.githubusercontent.com/stedolan/jq/master/sig/jq-release.key -O /tmp/jq-release.key && \
-    wget --no-check-certificate https://raw.githubusercontent.com/stedolan/jq/master/sig/v${JQ_VERSION}/jq-linux64.asc -O /tmp/jq-linux64.asc && \
-    wget --no-check-certificate https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 -O /tmp/jq-linux64 && \
-    gpg --import /tmp/jq-release.key && \
-    gpg --verify /tmp/jq-linux64.asc /tmp/jq-linux64 && \
-    cp /tmp/jq-linux64 /usr/bin/jq && \
-    chmod +x /usr/bin/jq && \
-    rm -f /tmp/jq-release.key && \
-    rm -f /tmp/jq-linux64.asc && \
-    rm -f /tmp/jq-linux64
+RUN yum update expat libxml2 gnupg2 libgcrypt openssl-libs pcre2 curl libcurl-minimal systemd platform-python python3-libs gnutls libksba sqlite-libs zlib libcom_err krb5-libs dbus libtasn1 libarchive unbound-libs tar platform-python-setuptools python3-setuptools-wheel sqlite-libs systemd systemd-libs systemd-pam oniguruma gnutls python3-rpm python3-unbound rpm rpm-build-libs rpm-libs rpm-plugin-systemd-inhibit unbound-libs -y
+RUN yum remove vim-minimal platform-python-pip.noarch -y
+
+RUN yum install -y jq-1.6
+RUN yum install -y which

Dockerfile-ciab

@@ -1,15 +1,20 @@
 # vim: ft=dockerfile
-ARG BASE_IMAGE
+# CHANGED FROM SOURCE: set the base image to fit the LOCALLY BUILT image
+# (which was built using `make`, based on my changes to the Makefile)
+ARG BASE_IMAGE="memsql/cluster-in-a-box"
 FROM ${BASE_IMAGE}
 
 # install SingleStore DB packages and additional dependencies
-ARG CLIENT_VERSION
-ARG SERVER_VERSION
-ARG STUDIO_VERSION
-ARG TOOLBOX_VERSION
+# CHANGED FROM SOURCE: switch to root, add specific versions according to Makefile
+USER root
+ARG CLIENT_VERSION="1.0.7"
+ARG SERVER_PACKAGE="singlestoredb-server"
+ARG SERVER_VERSION="8.1.31-d5850b922e"
+ARG STUDIO_VERSION="4.0.16"
+ARG TOOLBOX_VERSION="1.17.6"
 RUN yum install -y \
     singlestore-client-${CLIENT_VERSION} \
-    singlestoredb-server${SERVER_VERSION} \
+    ${SERVER_PACKAGE}${SERVER_VERSION} \
     singlestoredb-studio-${STUDIO_VERSION} \
     singlestoredb-toolbox-${TOOLBOX_VERSION} \
  && yum clean all
@@ -18,7 +23,11 @@ RUN mkdir -p /home/memsql && chown memsql:memsql /home/memsql
 ADD ciab-assets/studio.hcl /var/lib/singlestoredb-studio/studio.hcl
 RUN chown memsql:memsql /var/lib/singlestoredb-studio/studio.hcl
 
-VOLUME ["/var/lib/memsql"]
+# CHANGED FROM SOURCE: switch back from root
+USER memsql
+
+# CHANGED FROM SOURCE: commented the VOLUME line to enable persisting data in our automation-db-upload job
+#VOLUME ["/var/lib/memsql"]
 
 LABEL name="SingleStore DB Cluster in a Box"
 LABEL vendor="SingleStore"

Dockerfile-dynamic

@@ -5,6 +5,7 @@ FROM ${BASE_IMAGE}
 ARG CLIENT_VERSION
 RUN yum install -y \
     singlestore-client-${CLIENT_VERSION} \
+    java-11-openjdk \
  && yum clean all
 
 VOLUME ["/var/lib/memsql"]

Dockerfile-node

@@ -6,30 +6,20 @@ FROM ${BASE_IMAGE}
 ARG CLIENT_VERSION
 ARG SERVER_VERSION
 ARG LOCAL_SERVER_RPM
+ARG JRE_PACKAGES
 ADD assets /assets
 
 RUN if [[ -z "${LOCAL_SERVER_RPM}" ]] ; then \
       yum install -y memsql-server${SERVER_VERSION}; \
     else \
       rpm -i /assets/${LOCAL_SERVER_RPM}; \
     fi \
+ && for pkg in ${JRE_PACKAGES} ; do \
+      yum install -y ${pkg}; \
+    done \
  && yum install -y singlestore-client-${CLIENT_VERSION} \
  && yum clean all
 
-RUN yum install python39 -y && update-alternatives --set python /usr/bin/python3; if [ $? -ne 0 ]; then \
-wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
-    wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-runtime-2.0-1.el7.x86_64.rpm && \
-    wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-pip-9.0.1-5.el7.noarch.rpm && \
-    wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-3.6.12-1.el7.x86_64.rpm && \
-    wget http://mirror.centos.org/altarch/7/sclo/aarch64/rh/Packages/r/rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm && \
-    wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-python36-python-libs-3.6.12-1.el7.x86_64.rpm && \
-    yum install -y epel-release-latest-7.noarch.rpm rh-python36-runtime-2.0-1.el7.x86_64.rpm rh-python36-python-pip-9.0.1-5.el7.noarch.rpm rh-python36-python-3.6.12-1.el7.x86_64.rpm rh-python36-python-setuptools-36.5.0-1.el7.noarch.rpm rh-python36-python-libs-3.6.12-1.el7.x86_64.rpm;\
-    echo '/opt/rh/rh-python36/root/usr/lib64/' >> /etc/ld.so.conf && ldconfig;\
-    ln -s /opt/rh/rh-python36/root/usr/bin/python3 /usr/bin/python3;\
-    /usr/bin/python3 -m pip install --upgrade pip;\
-    /usr/bin/python3 -m pip install pymysql;\
-fi
-
 VOLUME ["/var/lib/memsql"]
 
 LABEL name="SingleStore DB Node"