Skip to content

sealer release v0.1.5 test report

Jump to bottom
Jiahui edited this page Jun 4, 2021 · 1 revision

CLOUD模式:

init:

1.wget https://github.com/alibaba/sealer/releases/download/v0.1.5-rc/sealer-0.0.0-linux-amd64.tar.gz && tar zxvf sealer-0.0.0-linux-amd64.tar.gz && mv sealer /usr/local/bin/
2.export 'ACCESSKEYID'='***'&&export 'ACCESSKEYSECRET'='***'&&export 'RegionID'='***'
3.sealer run registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9 集群成功启动

密码为run 镜像里Clusterfile里的密码

[root@iZ2vc2ce4hs85f7asxhaipZ ~]# cat /root/.sealer/my-cluster/Clusterfile
apiVersion: zlink.aliyun.com/v1alpha1
kind: Cluster
metadata:
  annotations:
    sea.aliyun.com/ClusterEIP: 47.108.132.245
    sea.aliyun.com/EipID: eip-2vca3x1lxu96ropt66am5
    sea.aliyun.com/Master0ID: i-2vc2ce4hs85f7ql6dqda
    sea.aliyun.com/Master0InternalIP: 172.16.0.194
    sea.aliyun.com/MasterIDs: i-2vc2ce4hs85f7ql6dqdb,i-2vc2ce4hs85f7ql6dqdc,i-2vc2ce4hs85f7ql6dqda
    sea.aliyun.com/NodeIDs: i-2vc4eamks4zejf9n16tm,i-2vc4eamks4zejf9n16tn,i-2vc4eamks4zejf9n16tl
    sea.aliyun.com/RegionID: cn-chengdu
    sea.aliyun.com/SecurityGroupID: sg-2vcdx512qke5iqpx1iwx
    sea.aliyun.com/VSwitchID: vsw-2vc55ufqm05qb8zdxux0h
    sea.aliyun.com/VpcID: vpc-2vcj16rdif2upvgeowdu6
    sea.aliyun.com/ZoneID: cn-chengdu-a
  creationTimestamp: null
  name: my-cluster
spec:
  certSANS:
  - aliyun-inc.com
  - 10.0.0.2
  image: kubernetes:v1.19.9
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.194
    - 172.16.0.196
    - 172.16.0.195
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.198
    - 172.16.0.199
    - 172.16.0.197
    memory: "4"
    systemDisk: "100"
  provider: ALI_CLOUD
  ssh:
    passwd: test123123
    pk: xxx
    pkPasswd: xxx
    user: root
status: {}

scale up:

1. 修改/root/.sealer/mycluster/Clusterfile文件中 Masters，Nodes Count数量：

  masters:
    count: "4"//master增加1个节点
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.194
    - 172.16.0.196
    - 172.16.0.195
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "4"//node增加1个节点

2. 执行 sealer apply -f /root/.sealer/my-cluster/Clusterfile (成功启动):

[root@iZ2vc2ce4hs85f7asxhaipZ ~]# kubectl get nodes
NAME                      STATUS   ROLES    AGE     VERSION
iz2vc1hagqsoc9ut8fp511z   Ready    <none>   2m50s   v1.19.9
iz2vc2ce4hs85f7ql6dqdaz   Ready    master   22m     v1.19.9
iz2vc2ce4hs85f7ql6dqdbz   Ready    master   20m     v1.19.9
iz2vc2ce4hs85f7ql6dqdcz   Ready    master   21m     v1.19.9
iz2vc4eamks4zejf9n16tlz   Ready    <none>   19m     v1.19.9
iz2vc4eamks4zejf9n16tmz   Ready    <none>   19m     v1.19.9
iz2vc4eamks4zejf9n16tnz   Ready    <none>   19m     v1.19.9
iz2vcdx512qke5j0l5bksrz   Ready    master   3m6s    v1.19.9
[root@iZ2vc2ce4hs85f7asxhaipZ ~]# cat /root/.sealer/my-cluster/Clusterfile
apiVersion: zlink.aliyun.com/v1alpha1
kind: Cluster
metadata:
  annotations:
    sea.aliyun.com/ClusterEIP: 47.108.132.245
    sea.aliyun.com/EipID: eip-2vca3x1lxu96ropt66am5
    sea.aliyun.com/Master0ID: i-2vc2ce4hs85f7ql6dqda
    sea.aliyun.com/Master0InternalIP: 172.16.0.194
    sea.aliyun.com/MasterIDs: i-2vc2ce4hs85f7ql6dqdb,i-2vc2ce4hs85f7ql6dqdc,i-2vc2ce4hs85f7ql6dqdai-2vcdx512qke5j0l5bksr
    sea.aliyun.com/NodeIDs: i-2vc4eamks4zejf9n16tm,i-2vc4eamks4zejf9n16tn,i-2vc4eamks4zejf9n16tli-2vc1hagqsoc9ut8fp511
    sea.aliyun.com/RegionID: cn-chengdu
    sea.aliyun.com/SecurityGroupID: sg-2vcdx512qke5iqpx1iwx
    sea.aliyun.com/VSwitchID: vsw-2vc55ufqm05qb8zdxux0h
    sea.aliyun.com/VpcID: vpc-2vcj16rdif2upvgeowdu6
    sea.aliyun.com/ZoneID: cn-chengdu-a
  creationTimestamp: null
  name: my-cluster
spec:
  certSANS:
  - aliyun-inc.com
  - 10.0.0.2
  image: kubernetes:v1.19.9
  masters:
    count: "4"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.194
    - 172.16.0.196
    - 172.16.0.195
    - 172.16.0.200
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "4"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.198
    - 172.16.0.199
    - 172.16.0.197
    - 172.16.0.201
    memory: "4"
    systemDisk: "100"
  provider: ALI_CLOUD
  ssh:
    passwd: test123123
    pk: xxx
    pkPasswd: xxx
    user: root
status: {}

scale down:

1. 修改/root/.sealer/mycluster/Clusterfile文件中 Masters，Nodes Count数量：

  masters:
    count: "3"//masters数量减1
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.194
    - 172.16.0.196
    - 172.16.0.195//iplist删去一个
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"//nodes数量减1
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.198
    - 172.16.0.199
    - 172.16.0.197//iplist删去一个
    memory: "4"
    systemDisk: "100"

2. 执行 sealer apply -f /root/.sealer/my-cluster/Clusterfile（执行成功）:

[root@iZ2vc2ce4hs85f7asxhaipZ ~]# sealer apply -f /root/.sealer/my-cluster/Clusterfile
2021-06-03 15:03:50 [INFO] reconcile master instances success [172.16.0.194 172.16.0.196 172.16.0.195]
2021-06-03 15:03:51 [INFO] reconcile node instances success [172.16.0.198 172.16.0.199 172.16.0.197]
2021-06-03 15:03:51 [INFO] desired master 3, current master 4, desired nodes 3, current nodes 4
2021-06-03 15:03:51 [INFO] delete nodes [172.16.0.200 172.16.0.201]
2021-06-03 15:03:51 [INFO] scale the cluster success
[root@iZ2vc2ce4hs85f7asxhaipZ ~]# kubectl get nodes
NAME                      STATUS   ROLES    AGE   VERSION
iz2vc2ce4hs85f7ql6dqdaz   Ready    master   39m   v1.19.9
iz2vc2ce4hs85f7ql6dqdbz   Ready    master   37m   v1.19.9
iz2vc2ce4hs85f7ql6dqdcz   Ready    master   38m   v1.19.9
iz2vc4eamks4zejf9n16tlz   Ready    <none>   37m   v1.19.9
iz2vc4eamks4zejf9n16tmz   Ready    <none>   37m   v1.19.9
iz2vc4eamks4zejf9n16tnz   Ready    <none>   37m   v1.19.9

3. 查看Clusterfile

[root@iZ2vc2ce4hs85f7asxhaipZ ~]# cat /root/.sealer/my-cluster/Clusterfile
apiVersion: zlink.aliyun.com/v1alpha1
kind: Cluster
metadata:
  annotations:
    ShouldBeDeleteInstancesIDs: ""
    sea.aliyun.com/ClusterEIP: 47.108.132.245
    sea.aliyun.com/EipID: eip-2vca3x1lxu96ropt66am5
    sea.aliyun.com/Master0ID: i-2vc2ce4hs85f7ql6dqda
    sea.aliyun.com/Master0InternalIP: 172.16.0.194
    sea.aliyun.com/MasterIDs: i-2vc2ce4hs85f7ql6dqdb,i-2vc2ce4hs85f7ql6dqdc,i-2vc2ce4hs85f7ql6dqdai-2vcdx512qke5j0l5bksr
    sea.aliyun.com/NodeIDs: i-2vc4eamks4zejf9n16tm,i-2vc4eamks4zejf9n16tn,i-2vc4eamks4zejf9n16tli-2vc1hagqsoc9ut8fp511
    sea.aliyun.com/RegionID: cn-chengdu
    sea.aliyun.com/SecurityGroupID: sg-2vcdx512qke5iqpx1iwx
    sea.aliyun.com/VSwitchID: vsw-2vc55ufqm05qb8zdxux0h
    sea.aliyun.com/VpcID: vpc-2vcj16rdif2upvgeowdu6
    sea.aliyun.com/ZoneID: cn-chengdu-a
  creationTimestamp: null
  name: my-cluster
spec:
  certSANS:
  - aliyun-inc.com
  - 10.0.0.2
  image: kubernetes:v1.19.9
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.194
    - 172.16.0.196
    - 172.16.0.195
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.198
    - 172.16.0.199
    - 172.16.0.197
    memory: "4"
    systemDisk: "100"
  provider: ALI_CLOUD
  ssh:
    passwd: Seadent123
    pk: xxx
    pkPasswd: xxx
    user: root
status: {}

4.缩小节点机器成功删除：

image.png

Bare metal ：

A.在master0上执行： 4C4G centos7.9

1.init：

1.wget https://github.com/alibaba/sealer/releases/download/v0.1.5-rc/sealer-0.0.0-linux-amd64.tar.g && tar zxvf sealer-0.0.0-linux-amd64.tar.gz && mv sealer /usr/local/bin/

2.sealer run registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9 --masters 172.16.0.165,172.16.0.166,172.16.0.167 --nodes 172.16.0.168,172.16.0.169 --passwd Seadent123
 集群成功启动
[root@test-baremetal004 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE   VERSION
test-baremetal003   Ready    master   12m   v1.19.9
test-baremetal004   Ready    master   13m   v1.19.9
test-baremetal005   Ready    master   12m   v1.19.9
test-baremetal006   Ready    <none>   11m   v1.19.9
test-baremetal007   Ready    <none>   11m   v1.19.9

存在问题：/root/.sealer/my-cluster/Clusterfile 文件中 Count字段数量没有变化（不影响使用）

[root@test-baremetal004 ~]# cat /root/.sealer/my-cluster/Clusterfile
apiVersion: zlink.aliyun.com/v1alpha1
kind: Cluster
metadata:
  creationTimestamp: null
  name: my-cluster
spec:
  certSANS:
  - aliyun-inc.com
  - 10.0.0.2
  image: registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.165
    - 172.16.0.166
    - 172.16.0.167
~
  nodes:
    count: "3" //count 与iplist数量不匹配
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.168
    - 172.16.0.169
~

2.scale up

**添加节点 masters : 172.16.0.170，172.16.0.171；nodes:**172.16.0.172

  1. 修改 Clusterfile：
~
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.165
    - 172.16.0.166
    - 172.16.0.167
    - 172.16.0.170//masters增加2节点
    - 172.16.0.171
~
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.168
    - 172.16.0.169
    - 172.16.0.172//nodes增加1节点
  1. 执行sealer apply -f /root/.sealer/my-cluster/Clusterfile （执行成功）：
[root@test-baremetal004 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE   VERSION
test-baremetal001   Ready    <none>   33s   v1.19.9
test-baremetal002   Ready    master   56s   v1.19.9
test-baremetal003   Ready    master   26m   v1.19.9
test-baremetal004   Ready    master   27m   v1.19.9
test-baremetal005   Ready    master   25m   v1.19.9
test-baremetal006   Ready    <none>   25m   v1.19.9
test-baremetal007   Ready    <none>   25m   v1.19.9
test-baremetal009   Ready    master   74s   v1.19.9

3. scale down

  1. 修改Clusterfile：
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.165
    - 172.16.0.166
    - 172.16.0.167
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.168
    - 172.16.0.169
  1. 执行sealer apply -f /root/.sealer/my-cluster/Clusterfile（执行成功）：
[root@test-baremetal004 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE   VERSION
test-baremetal003   Ready    master   29m   v1.19.9
test-baremetal004   Ready    master   29m   v1.19.9
test-baremetal005   Ready    master   28m   v1.19.9
test-baremetal006   Ready    <none>   27m   v1.19.9
test-baremetal007   Ready    <none>   27m   v1.19.9

** **

问题记录：

//scale down后直接执行sealer delete -f Clusterfile 那么scale down删掉的两个节点服务器内仍然存在rootfs目录 和seatutil工具

[root@baremetal002 ~]# ssh [email protected]
[email protected]'s password:
Last login: Thu Jun  3 17:08:30 2021 from 172.16.0.156
Welcome to Alibaba Cloud Elastic Compute Service !

[root@iZ2vcdx512qke5k9yuwucfZ ~]# ls /var/lib/sealer/data/my-cluster/rootfs/
bin  cri  images                    Kubefile  README.md  scripts  TREE.md
cni  etc  kubeadm-join-config.yaml  Metadata  registry   statics

4.delete

执行sealer delete -f /root/.sealer/my-cluster/Clusterfile（最后一步报错,不影响结果）:

各节点成功删除

/bin/sh: /var/lib/sealer/data/my-cluster/rootfs/scripts/clean.sh: No such file or directory
2021-06-04 15:53:54 [EROR] exec command failed Process exited with status 127
2021-06-04 15:53:54 [EROR] 172.16.0.165:exec /bin/sh -c /var/lib/sealer/data/my-cluster/rootfs/scripts/clean.sh failed, exec command failed 172.16.0.165 /bin/sh -c /var/lib/sealer/data/my-cluster/rootfs/scripts/clean.sh
2021-06-04 15:53:55 [INFO] [ssh][172.16.0.169] : rm -rf /var/lib/sealer/data/my-cluster/rootfs
2021-06-04 15:53:55 [INFO] [ssh][172.16.0.168] : rm -rf /var/lib/sealer/data/my-cluster/rootfs
2021-06-04 15:53:55 [INFO] [ssh][172.16.0.167] : rm -rf /var/lib/sealer/data/my-cluster/rootfs
2021-06-04 15:53:55 [INFO] [ssh][172.16.0.166] : rm -rf /var/lib/sealer/data/my-cluster/rootfs
2021-06-04 15:53:55 [EROR] unmountRootfs failed

//EROR原因：/var/lib/sealer/data/my-cluster该目录不存在

**//：**节点Clean后seautil工具仍存在

5.delete后重新run：

  1. **执行：*sealer run registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9 --masters 172.16.0.165,172.16.0.166,172.16.0.167 --nodes 172.16.0.168,172.16.0.169 --passwd ** (成功执行） 
[root@baremetal002 ~]# kubectl get nodes -A
NAME           STATUS   ROLES    AGE     VERSION
baremetal001   Ready    <none>   46s     v1.19.9
baremetal002   Ready    master   2m21s   v1.19.9
baremetal003   Ready    master   98s     v1.19.9
baremetal004   Ready    <none>   45s     v1.19.9
  1. sealer delete -f /root/.sealer/my-cluster/Clusterfile

B.不在master0执行： 4C4G centos7.9

masters 172.16.0.165,172.16.0.166,172.16.0.167
nodes 172.16.0.168,172.16.0.169

在172.16.0.164节点执行：

sealer run registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9

--masters 172.16.0.165,172.16.0.166,172.16.0.167 --nodes 172.16.0.168,172.16.0.169 --passwd ** （成功执行）

[root@test-baremetal008 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE    VERSION
test-baremetal003   Ready    master   87s    v1.19.9
test-baremetal004   Ready    master   2m6s   v1.19.9
test-baremetal005   Ready    master   33s    v1.19.9
test-baremetal006   Ready    <none>   15s    v1.19.9
test-baremetal007   Ready    <none>   14s    v1.19.9

2.scale up

masters添加节点172.16.0.170，172.16.0.171

nodes添加节点172.16.0.172

  1. 修改 Clusterfile：
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.165
    - 172.16.0.166
    - 172.16.0.167
        - 172.16.0.170//增加节点
    - 172.16.0.171//增加节点
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.168
    - 172.16.0.169
    - 172.16.0.172//增加节点
  1. 执行sealer apply -f /root/.sealer/my-cluster/Clusterfile （执行成功）

image.png

[root@test-baremetal008 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE     VERSION
test-baremetal001   Ready    <none>   117s    v1.19.9
test-baremetal002   Ready    master   2m14s   v1.19.9
test-baremetal003   Ready    master   8m58s   v1.19.9
test-baremetal004   Ready    master   9m37s   v1.19.9
test-baremetal005   Ready    master   8m4s    v1.19.9
test-baremetal006   Ready    <none>   7m46s   v1.19.9
test-baremetal007   Ready    <none>   7m45s   v1.19.9
test-baremetal009   Ready    master   2m52s   v1.19.9

3. scale down

​ 删除172.16.0.170，172.16.0.171 master 节点和 172.16.0.172 node 节点

  1. 修改Clusterfile：
  masters:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.165
    - 172.16.0.166
    - 172.16.0.167  //删去172.16.0.170，172.16.0.171 master节点
    memory: "4"
    systemDisk: "100"
  network:
    cniName: calico
    interface: eth0
    podCIDR: 100.64.0.0/10
    svcCIDR: 10.96.0.0/22
  nodes:
    count: "3"
    cpu: "4"
    dataDisks:
    - "100"
    ipList:
    - 172.16.0.168
    - 172.16.0.169  //删去172.16.0.172 node节点
  1. 执行sealer apply -f /root/.sealer/my-cluster/Clusterfile（执行成功）：
[root@test-baremetal008 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE   VERSION
test-baremetal003   Ready    master   16m   v1.19.9
test-baremetal004   Ready    master   16m   v1.19.9
test-baremetal005   Ready    master   15m   v1.19.9
test-baremetal006   Ready    <none>   14m   v1.19.9
test-baremetal007   Ready    <none>   14m   v1.19.9

问题记录（同master0节点执行scale down中的问题）：

scale down 节点需要删除rootfs以及seautil工具

[root@test-baremetal008 ~]# ssh [email protected] //进入 已经scale down掉的节点
Welcome to Alibaba Cloud Elastic Compute Service !
[root@test-baremetal009 ~]# ls /var/lib/sealer/data/my-cluster/rootfs/ 
bin/                      cri/                      images/                   Kubefile                  README.md                 scripts/                  TREE.md                   
cni/                      etc/                      kubeadm-join-config.yaml  Metadata                  registry/                 statics/

4.delete

执行 ： sealer delete -f /root/.sealer/my-cluster/Clusterfile （成功删除没用报错）：

5.delete后重新run：

  1. 执行： **sealer run registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:v1.19.9 --masters 172.16.0.165,172.16.0.166,172.16.0.167 --nodes 172.16.0.168,172.16.0.169 --passwd **（成功执行） 
[root@test-baremetal008 ~]# kubectl get nodes -A
NAME                STATUS   ROLES    AGE   VERSION
test-baremetal003   Ready    master   12m   v1.19.9
test-baremetal004   Ready    master   12m   v1.19.9
test-baremetal005   Ready    master   11m   v1.19.9
test-baremetal006   Ready    <none>   11m   v1.19.9
test-baremetal007   Ready    <none>   11m   v1.19.9
Clone this wiki locally