redis-set.yaml

#       Copyright 2020, noelmcloughlin
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

### If you wait to do everything until you're sure it's right,
### you'll probably never do much of anything” ~ Win Borden


# STORAGE CLASS
# ... delay volume binding until pods are scheduled
# refs: 
#   https://kubernetes.io/docs/concepts/storage/storage-classes/#local
#   https://kubernetes.io/blog/2018/04/13/local-persistent-volumes-beta/
#   https://github.com/lusyoe/k8s-local-storage
#   https://suraj.pro/post/hostmount-minikube/

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: local
  labels:
    type: local
    tier: infra
  annotations:
    storageclass.beta.kubernetes.io/is-default-class: "true"
#provisioner: kubernetes.io/no-provisioner
provisioner: k8s.io/minikube-hostpath
parameters:
  pool: default
  description: Kubernetes volume
  fsType: ext4
  adminSecretNamespace: default
  adminSecretName: localos-secret
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: false
---
# PERSISTENT VOLUMES
# refs:
#   https://kubernetes.io/docs/concepts/storage/volumes/#local
#   https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage
#   https://kubernetes.io/blog/2018/04/13/local-persistent-volumes-beta/
#   https://akomljen.com/kubernetes-persistent-volumes-with-deployment-and-statefulset/
#   https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1
#   https://github.com/lusyoe/k8s-local-storage

apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv1
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv1
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv2
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv2
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv3
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv3
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv4
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv4
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv5
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv5
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: redis-pv6
  labels:
    app: redis
    bind: redis
    tier: infra
  annotations:
    #pv.beta.kubernetes.io/gid: "1234"
    "volume.alpha.kubernetes.io/node-affinity": '{
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          { "matchExpressions": [
              { "key": "kubernetes.io/hostname",
                "operator": "In",
                "values": ["minikube"]
              }
          ]}
        ]}}'
spec:
  capacity:
    storage: 500Mi
  accessModes:
    - ReadWriteOnce
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Delete #Retain
  storageClassName: local
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv6
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
  hostPath:
    type: DirectoryOrCreate
    path: /tmp/redis-pv6
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - minikube
---
#SERVICE
# refs:
#   https://schoolofdevops.github.io/ultimate-kubernetes-bootcamp/13_redis_statefulset
#   https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/#deploy-redis
#   https://github.com/sobotklp/kubernetes-redis-cluster/blob/master/redis-cluster.yml
#   https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#components
#   https://kubernetes.io/docs/concepts/mnt/config-maps-networking/dns-pod-service/#a-records

## Headless service for stable dns names
apiVersion: v1
kind: Service
metadata:
  name: redis
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "false"
  labels:
    app: redis
spec:
  ports:
  - name: redis
    port: 6379
    targetPort: redis
  - port: 16379
    targetPort: 16379
    name: gossip
#  type: ClusterIp
  selector:
    app: redis
    tier: backend
    role: master
---
# CONFIGMAPS
# refs:
#    https://raw.githubusercontent.com/kubernetes/examples/master/staging/storage/redis/image/redis-master.conf
#    https://raw.githubusercontent.com/kubernetes/examples/master/staging/storage/redis/image/redis-slave.conf
#    https://schoolofdevops.github.io/ultimate-kubernetes-bootcamp/13_redis_statefulset

apiVersion: v1
kind: ConfigMap
metadata:
  name: env-config
  namespace: default
data:
  REDIS_HOST: 'redis.default.svc.cluster.local'
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis
data:
  # Apply this config only on the master.
  master.conf: |-
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file nodes.conf
    cluster-migration-barrier 1
    appendonly yes
    # Other cluster members need to be able to connect
    protected-mode no

  # Apply this config only on the slaves.
  slave.conf: |-
    cluster-enabled yes
    cluster-require-full-coverage no
    cluster-node-timeout 15000
    cluster-config-file nodes.conf
    cluster-migration-barrier 1
    appendonly yes
    # Other cluster members need to be able to connect
    protected-mode no

# refs:
#   https://schoolofdevops.github.io/ultimate-kubernetes-bootcamp/13_redis_statefulset
#   https://github.com/sobotklp/kubernetes-redis-cluster/blob/master/redis-cluster.yml

  init.sh: |+
    set -e
    ###############################################################
    ## Bootstrap Stateful Set members (requires /etc/podinfo)
    ###############################################################
    if [ -f /etc/podinfo/pod_name ]; then
        ORDINAL="$( cat /etc/podinfo/pod_name | rev | cut -d- -f1 )"
    else
        ORDINAL=1 && [ ${HOSTNAME} == 'redis-0' ] && ORDINAL=0
    fi

        ## Copy service config from config-map to emptyDir
    if [ "$ORDINAL" -eq "0" ];then
       cp /mnt/config-map/master.conf /mnt/conf.d/redis.conf
    else
       cp /mnt/config-map/slave.conf /mnt/conf.d/redis.conf
    fi
    cp /mnt/config-map/run.sh /mnt/conf.d/run.sh || true

    ## peer-finder tool
    if [ -x /bin/wget ]; then
        wget https://storage.googleapis.com/kubernetes-release/pets/peer-finder -O /mnt/conf.d/peer-finder
        chmod u+x /mnt/conf.d/peer-finder
    fi
    ### If you wait to do everything until you're sure it's right,
    ### you'll probably never do much of anything” ~ Win Borden
    wait

  run.sh: |+
    set -e
    ##############################################################
    ## Start service, discover peers, join cluster (good luck)
    ##############################################################
    /usr/local/bin/redis-server /etc/redis/conf.d/redis.conf &
    sleep 2
    if [ -x /etc/redis/conf.d/peer-finder ]; then
        /etc/redis/conf.d/peer-finder -on-start 'tee > /etc/redis/initial_peers' -service redis-cluster -ns $POD_NAMESPACE
    fi
    if [ -f /etc/podinfo/pod_name ]; then
        ORDINAL="$( cat /etc/podinfo/pod_name | rev | cut -d- -f1 )"
    else
        ORDINAL=1 && [ "${HOSTNAME}" == 'redis-0' ] && ORDINAL=0
    fi
    if [ "$ORDINAL" -eq "0" ]; then
        ## Controllers
        /usr/local/bin/redis-cli cluster addslots $(seq 0 16383)
    else
        # Slaves of a random master
        PEER_IP="$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1')"
        if [ -n "${PEER_IP}" ]; then
            /usr/local/bin/redis-cli cluster meet ${PEER_IP} 6379
            sleep 1 
            MASTER_ID="$( redis-cli --csv cluster slots | cut -d, -f 5 | sed -e 's/^"//'  -e 's/"$//' )"
            /usr/local/bin/redis-cli cluster replicate $MASTER_ID
        fi
    fi
    ### If you wait to do everything until you're sure it's right,
    ### you'll probably never do much of anything” ~ Win Borden
    wait
---
#SECRETS
# refs:
#    https://kubernetes.io/docs/concepts/configuration/secret/#creating-a-secret-manually

apiVersion: v1
kind: Secret
metadata:
  name: redis
type: Opaque
data:
  REDIS_USER: YWRtaW4=
  REDIS_PASS: YWRtaW4=
stringData:
  REDIS_USER: admin
  REDIS_PASS: admin
#---
#apiVersion: v1
#kind: Secret
#metadata:
#  name: dotfile-secret
#data:
#  .secret-file: dmFsdWUtMg0KDQo=
#---
#apiVersion: v1
#kind: Pod
#metadata:
#  name: secret-dotfiles-pod
#spec:
#  volumes:
#  - name: secret-volume
#    secret:
#      secretName: dotfile-secret
#  containers:
#  - name: dotfile-test-container
#    image: k8s.gcr.io/busybox
#    command:
#    - ls
#    args:
#    - "-l"
#    - "/etc/secret-volume"
#    volumeMounts:
#    - name: secret-volume
#      readOnly: true
#      mountPath: "/etc/secret-volume"
---
# STATEFULSET
# refs:
#    https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset
#    https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#using-statefulsets
#    https://kubernetes.io/docs/tutorials/stateful-application/cassandra/#using-a-statefulset-to-create-a-cassandra-ring
#    https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
#    https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
#    https://akomljen.com/kubernetes-persistent-volumes-with-deployment-and-statefulset/
#    https://raw.githubusercontent.com/kubernetes/examples/master/staging/storage/redis/redis-master.yaml
#    https://estl.tech/deploying-redis-with-persistence-on-google-kubernetes-engine-c1d60f70a043
#    https://kubernetes.io/docs/tutorials/configuration/configure-redis-using-configmap/

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  labels:
    app: redis
    tier: backend
spec:
  serviceName: redis
  replicas: 3
  podManagementPolicy: Parallel
  selector:
    matchLabels:
      app: redis
      tier: backend
  template:
    metadata:
      labels:
        app: redis
        tier: backend
    spec:
      terminationGracePeriodSeconds: 5
      initContainers:
      - name: init
        image: redis:3.2.10-alpine
        command:
        - sh
        - /mnt/config-map/init.sh
        volumeMounts:
        - name: conf
          mountPath: /mnt/conf.d
          readOnly: false
        - name: config-map
          mountPath: /mnt/config-map
        - name: podinfo
          mountPath: /etc/podinfo
          readOnly: false

      containers:
      - name: redis
        image: redis:3.2.10-alpine
        #imagePullPolicy: Always
        command:
        - sh
        - /etc/redis/conf.d/run.sh
        volumeMounts:
        - name: conf
          mountPath: /etc/redis/conf.d
          readOnly: false
        - name: data
          mountPath: /var/lib/redis
          subPath: redis
          readOnly: false
        env:
        - name: ALLOW_EMPTY_PASSWORD
          value: "yes"
        - name: REDIS_PASS
          valueFrom:
            secretKeyRef:
              name: redis
              key: REDIS_PASS
        - name: REDIS_HOST
          valueFrom:
            configMapKeyRef:
              name: env-config
              key: REDIS_HOST
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        ports:
        - name: redis
          containerPort: 6379
        resources:
          limits:
            memory: 500Mi
          requests:
            memory: 500Mi
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - '-c'
              - echo 'goodbye world'
        livenessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - "redis-cli -h $(hostname) ping"
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        readinessProbe:
          exec:
            # Check we can execute queries over TCP (skip-networking is off).
            command: ["redis", "-h", "127.0.0.1", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 2
          timeoutSeconds: 1
          failureThreshold: 3

# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
# https://kubernetes.io/blog/2018/04/13/local-persistent-volumes-beta/
# https://kubernetes.io/blog/2018/10/11/topology-aware-volume-provisioning-in-kubernetes/

      volumes:
      - name: conf
        emptyDir: {}
      - name: config-map
        configMap:
          name: redis
      - name: podinfo
        ## init containers may use this to find cluster peers
        downwardAPI:
          items:
            - path: "labels"
              fieldRef:
                fieldPath: metadata.labels
            - path: "annotations"
              fieldRef:
                fieldPath: metadata.annotations
            - path: "pod_name"
              fieldRef:
                fieldPath: metadata.name
            - path: "pod_namespace"
              fieldRef:
                fieldPath: metadata.namespace

  volumeClaimTemplates:
    - metadata:
        name: data
        annotations:
          volume.beta.kubernetes.io/storage-class: local
      spec:
        storageClassName: local
        accessModes: [ "ReadWriteOnce" ]
        resources:
          requests:
            storage: 500m
        selector:
          matchLabels:
            bind: redis