bb.org: fix galera timeout

Buildbot seems to enter systematically in timeout if it is not able to pull logs from the libvirt worker. So, it's mandatory to fetch logs from a readable and browsable (+x) PATH by the buildbot user (/var/log/syslog is not working for instance). Plus use of lazylogfiles that seems cleaner, see https://docs.buildbot.net/current/manual/configuration/steps/shell_command.html.
MariaDB · May 9, 2022 · cd5321a · cd5321a
1 parent 36b08a0
commit cd5321a
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 10 deletions.
diff --git a/buildbot.mariadb.org/master-libvirt/master.cfg b/buildbot.mariadb.org/master-libvirt/master.cfg
@@ -111,7 +111,7 @@ def getRpmUpgradeStep():
         descriptionDone=["test", "upgrade"],
         env=envFromProperties(['test_mode', 'test_type', 'branch', 'major_version', 'arch', 'systemdCapability', 'is_main_tree', 'version_name']),
         command=['./rpm-upgrade.sh'])
-	
+
 def getRpmInstallStep():
      return Test(
         name="install",
@@ -147,7 +147,23 @@ def getDebGaleraStep(port):
         description=["testing", "galera", "SST"],
         descriptionDone=["galera", "SST"],
         timeout=300,
-        logfiles={"daemon": "/var/log/daemon.log", "syslog": "/var/log/syslog", "node1": "/var/lib/node1/node1.err", "node2": "/var/lib/node2/node2.err", "node3": "/var/lib/node3/node3.err", "mysqld.1.err": "/home/buildbot/var/log/mysqld.1.err", "mysqld.2.err": "/home/buildbot/var/log/mysqld.2.err", "mysqld.3.err": "/home/buildbot/var/log/mysqld.3.err", "node1.mariabackup.prepare": "/home/buildbot/mariabackup_logs/node1.mariabackup.prepare.log", "node2.mariabackup.prepare": "/home/buildbot/mariabackup_logs/node2.mariabackup.prepare.log", "node3.mariabackup.prepare": "/home/buildbot/mariabackup_logs/node3.mariabackup.prepare.log", "node1.mariabackup.move": "/home/buildbot/mariabackup_logs/node1.mariabackup.move.log", "node2.mariabackup.move": "/home/buildbot/mariabackup_logs/node2.mariabackup.move.log", "node3.mariabackup.move": "/home/buildbot/mariabackup_logs/node3.mariabackup.move.log", "node1.mariabackup.backup": "/home/buildbot/mariabackup_logs/node1.mariabackup.backup.log", "node2.mariabackup.backup": "/home/buildbot/mariabackup_logs/node2.mariabackup.backup.log", "node4.mariabackup.backup": "/home/buildbot/mariabackup_logs/node4.mariabackup.backup.log"},
+        lazylogfiles=True,
+        logfiles={
+          "daemon": "/home/buildbot/logs/daemon.log",
+          "syslog": "/home/buildbot/logs/syslog",
+          "node1": "/home/buildbot/logs/node1.err",
+          "node2": "/home/buildbot/logs/node2.err",
+          "node3": "/home/buildbot/logs/node3.err",
+          "node1.mariabackup.prepare": "/home/buildbot/logs/node1.mariabackup.prepare.log",
+          "node2.mariabackup.prepare": "/home/buildbot/logs/node2.mariabackup.prepare.log",
+          "node3.mariabackup.prepare": "/home/buildbot/logs/node3.mariabackup.prepare.log",
+          "node1.mariabackup.move": "/home/buildbot/logs/node1.mariabackup.move.log",
+          "node2.mariabackup.move": "/home/buildbot/logs/node2.mariabackup.move.log",
+          "node3.mariabackup.move": "/home/buildbot/logs/node3.mariabackup.move.log",
+          "node1.mariabackup.backup": "/home/buildbot/logs/node1.mariabackup.backup.log",
+          "node2.mariabackup.backup": "/home/buildbot/logs/node2.mariabackup.backup.log",
+          "node4.mariabackup.backup": "/home/buildbot/logs/node4.mariabackup.backup.log"
+        },
         doStepIf=if_run_galera_test,
         env=envFromProperties(['version_name', 'master_branch', 'sst_mode', 'arch']),
         command=["./deb-galera.sh"]

diff --git a/buildbot.mariadb.org/scripts/deb-galera.sh b/buildbot.mariadb.org/scripts/deb-galera.sh
@@ -14,24 +14,28 @@ set -e
 store_logs() {
   # Make sure we store all existing logs, whenever we decide to exit game
   set +e
-  mkdir /home/buildbot/sst_logs
+  mkdir /home/buildbot/logs
+  for log in /var/log/daemon.log /var/log/syslog; do
+    [[ -f $log ]] && sudo cp $log /home/buildbot/logs
+  done
   # It looks like buildbot may be enveloping the path into single quotes
   # if it has wildcards, as in '/var/lib/node*/node*.err'.
   # Trying to get rid of the wildcards
-  sudo chmod uga+r /var/lib/node1/node1.err /var/lib/node2/node2.err /var/lib/node3/node3.err
-  sudo cp /var/lib/node1/node1.err /var/lib/node2/node2.err /var/lib/node3/node3.err /home/buildbot/sst_logs
+  for node in 1 2 3; do
+    sudo cp /var/lib/node${node}/node${node}.err /home/buildbot/logs
+  done
   if [[ $sst_mode == "mariabackup" ]]; then
-    mkdir /home/buildbot/sst_logs/mbackup
     for node in 1 2 3; do
       for log in prepare move backup; do
         if [[ -f /var/lib/node${node}/mariabackup.${log}.log ]]; then
-          sudo cp /var/lib/node${node}/mariabackup.${log}.log /home/buildbot/sst_logs/mbackup/node${node}.mariabackup.${log}.log
+          sudo cp /var/lib/node${node}/mariabackup.${log}.log /home/buildbot/logs/node${node}.mariabackup.${log}.log
         fi
       done
     done
-    sudo chown -R buildbot:buildbot /home/buildbot/sst_logs
+    sudo chown -R buildbot:buildbot /home/buildbot/logs
   fi
-  ls -l /home/buildbot/sst_logs/
+  sudo chmod -R +r /home/buildbot/logs
+  ls -l /home/buildbot/logs
 }
 
 # function to be able to run the script manually (see bash_lib.sh)
@@ -281,7 +285,7 @@ done
 
 bb_log_info "stop cluster"
 sudo killall -s 9 mariadbd || true
-sleep 1
+sudo killall -s 9 mysqld || true
 sudo killall -s 9 mysqld_safe || true
 
 bb_log_ok "all done"