-
Notifications
You must be signed in to change notification settings - Fork 2
/
rolling-chunkserver.yml
147 lines (128 loc) · 6.78 KB
/
rolling-chunkserver.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
---
# Performs a rolling restart on the chunkservers one by one and waits for
# each to come up before moving to the next.
#
# By default it only upgrades chunkservers that are healthy and out of date.
- name: Restart chunkservers
hosts: moosefs_chunkserver
serial: 1
become: true
vars_files:
- "vars/moosefs-settings.yml"
- "vars/upgrade-settings.yml"
pre_tasks:
- name: Get MooseFS version
shell: "mfschunkserver -v | cut -d' ' -f 2 | cut -d'-' -f 1"
register: moosefs_chunkserver_version
check_mode: false
changed_when: false
- name: Set MooseFS version
set_fact:
moosefs_version: "{{ moosefs_chunkserver_version.stdout }}"
- name: Check the state of the chunk matrix before any changes
command: mfscli -SIC -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunks_status
check_mode: false
changed_when: false
- name: Check the list of all connected chunkservers using mfscli
command: mfscli -SCS -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunkserver_status
check_mode: false
changed_when: false
- name: Determine the current state of the chunk matrix
set_fact:
moosefs_chunks_missing: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass missing:') | select('search', ': 0') | length == 0 }}"
moosefs_chunks_endangered: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass endangered:') | select('search', ': 0') | length == 0 }}"
moosefs_chunks_undergoal: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass undergoal:') | select('search', ': 0') | length == 0 }}"
- name: Determine the current state of the chunkserver
set_fact:
moosefs_chunkserver_connected: "{{ moosefs_chunkserver_status.stdout_lines | select('search', inventory_hostname) | length != 0 }}"
moosefs_chunkserver_version_match: "{{ moosefs_chunkserver_status.stdout_lines | select('search', inventory_hostname) | select('search', moosefs_version) | length != 0 }}"
moosefs_chunkserver_upgrade_needed: "{{ moosefs_chunkserver_status.stdout_lines | select('search', inventory_hostname) | select('search', moosefs_version) | length == 0 }}"
- name: Print the current state of the chunk matrix
debug:
msg:
- "Chunks missing? {{ moosefs_chunks_missing }}"
- "Chunks endangered? {{ moosefs_chunks_endangered }}"
- "Chunks undergoal? {{ moosefs_chunks_undergoal }}"
- name: Print the current state of the chunkserver
debug:
msg:
- "Connected? {{ moosefs_chunkserver_connected }}"
- "Version matches target? {{ moosefs_chunkserver_version_match }}"
- "Upgrade needed? {{ moosefs_chunkserver_upgrade_needed }}"
- name: Give some feedback as to what will happen next
debug:
msg:
- "Abort due to missing chunks: {{ moosefs_chunks_missing and moosefs_abort_on_missing_chunks }}"
- "Abort due to endangered chunks: {{ moosefs_chunks_endangered and moosefs_abort_on_endangered_chunks }}"
- "Abort due to undergoal chunks: {{ moosefs_chunks_undergoal and moosefs_abort_on_undergoal_chunks }}"
- name: Abort if we hit a failure condition
assert:
that:
- "moosefs_chunks_missing == false or moosefs_abort_on_missing_chunks == false"
- "moosefs_chunks_endangered == false or moosefs_abort_on_endangered_chunks == false"
- "moosefs_chunks_undergoal == false or moosefs_abort_on_undergoal_chunks == false"
success_msg: "All assertions passed, no abort condition found."
fail_msg: "ERROR: We had to abort. Please read the above logs for more details."
- name: Wait until zero missing chunks
command: mfscli -SIC -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunks_status
check_mode: false
changed_when: false
retries: 1800
until: "moosefs_chunks_status.stdout_lines | select('search', 'chunkclass missing:') | select('search', ': 0') | length != 0"
- name: Wait until zero endangered chunks
command: mfscli -SIC -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunks_status
check_mode: false
changed_when: false
retries: 1800
until: "moosefs_chunks_status.stdout_lines | select('search', 'chunkclass endangered:') | select('search', ': 0') | length != 0"
- name: Wait until zero undergoal chunks
command: mfscli -SIC -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunks_status
check_mode: false
changed_when: false
retries: 1800
until: "moosefs_chunks_status.stdout_lines | select('search', 'chunkclass undergoal:') | select('search', ': 0') | length != 0"
tasks:
- name: Restart chunkserver service
service:
name: moosefs-chunkserver
state: restarted
when:
- not moosefs_pro
- moosefs_restart_chunkservers
- moosefs_chunkserver_upgrade_needed or moosefs_restart_chunkservers_force
- not (moosefs_chunks_missing and moosefs_abort_on_missing_chunks)
- not (moosefs_chunks_endangered and moosefs_abort_on_endangered_chunks)
- not (moosefs_chunks_undergoal and moosefs_abort_on_undergoal_chunks)
- name: Restart chunkserver service (PRO)
service:
name: moosefs-pro-chunkserver
state: restarted
when:
- moosefs_pro
- moosefs_restart_chunkservers
- moosefs_chunkserver_upgrade_needed or moosefs_restart_chunkservers_force
- not (moosefs_chunks_missing and moosefs_abort_on_missing_chunks)
- not (moosefs_chunks_endangered and moosefs_abort_on_endangered_chunks)
- not (moosefs_chunks_undergoal and moosefs_abort_on_undergoal_chunks)
post_tasks:
- name: Check the state of the chunk matrix after our run
command: mfscli -SIC -H {{ moosefs_master_host }} -p -s ' '
register: moosefs_chunks_status
check_mode: false
changed_when: false
- name: Determine the current state of the chunk matrix
set_fact:
moosefs_chunks_missing: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass missing:') | select('search', ': 0') | length == 0 }}"
moosefs_chunks_endangered: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass endangered:') | select('search', ': 0') | length == 0 }}"
moosefs_chunks_undergoal: "{{ moosefs_chunks_status.stdout_lines | select('search', 'chunkclass undergoal:') | select('search', ': 0') | length == 0 }}"
- name: Print the current state of the chunk matrix
debug:
msg:
- "Chunks missing? {{ moosefs_chunks_missing }}"
- "Chunks endangered? {{ moosefs_chunks_endangered }}"
- "Chunks undergoal? {{ moosefs_chunks_undergoal }}"