Merge pull request #23 from jordiprats/master

millores varies

.gitignore

@@ -7,3 +7,6 @@ junit
 log
 spec/fixtures/
 Gemfile.lock
+/.yardwarns
+/.yardoc
+/doc

CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+## 0.4.13
+
+* added **sql_mode** and **lower_case_table_names** to **mysqld section**
+* fixed **unless** and **onlyif** for **mysql_sql**
+* moved backup scripts to files folder
+* minor bugfix mysqldump script
+* added **masterdata** and **file_per_db** options to **mysql::backup::mysqldump**
+
+## 0.4.12
+
+* bugfix **backup_xtrabackup** script (no cleanup was performed when MAILTO was undefined)
+
 ## 0.4.11
 
 * added **mysql::tools::perconatoolkit**

Rakefile

@@ -4,8 +4,9 @@ require 'puppet-lint/tasks/puppet-lint'
 require 'metadata-json-lint/rake_task'
 PuppetLint.configuration.send('disable_80chars')
 PuppetLint.configuration.send('disable_140chars')
-PuppetLint.configuration.send('disable_class_inherits_from_params_class')
 PuppetLint.configuration.send('disable_documentation')
+PuppetLint.configuration.send('disable_arrow_on_right_operand_line')
+PuppetLint.configuration.send('disable_class_inherits_from_params_class')
 PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp"]
 
 desc "Validate manifests, templates, and ruby files"

templates/backup/mysqldump/backupmysqldump.erb → files/backup/mysqldump/backupmysqldump.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#TODO: custom conf
+# puppet managed file
 
 function initbck
 {
@@ -13,16 +13,17 @@ function initbck
 		mkdir -p $DESTINATION
 		BACKUPTS=$(date +%Y%m%d%H%M)
 
-		CURRENTBACKUPLOG="$LOGDIR/$BACKUPTS.log"
-
-		BCKFAILED=0
-
-		if [ -z "$LOGDIR" ];
+		if [ -z "${LOGDIR}" ];
 		then
 			exec 2>&1
+			LOGDIR=$DESTINATION
 		else
 			exec >> $CURRENTBACKUPLOG 2>&1
 		fi
+
+		CURRENTBACKUPLOG="$LOGDIR/$BACKUPTS.log"
+
+		BCKFAILED=0
 	fi
 }
 
@@ -65,7 +66,7 @@ function dump_grants
 
 	GRANTSDESTFILE="$GRANTSDEST/${IDHOST}.grants.sql"
 
-	echo "SELECT DISTINCT CONCAT('SHOW GRANTS FOR ''', user, '''@''', host, ''';') FROM mysql.user" | $MYSQLBIN --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf -N | $MYSQLBIN --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf -N 2>/dev/null | sed 's/$/;/' > $GRANTSDESTFILE
+	echo "SELECT DISTINCT CONCAT('SHOW GRANTS FOR ''', user, '''@''', host, ''';') FROM mysql.user" | $MYSQLBIN ${MYSQL_INSTANCE_OPTS} -N | $MYSQLBIN ${MYSQL_INSTANCE_OPTS} -N 2>/dev/null | sed 's/$/;/' > $GRANTSDESTFILE
 
 	if [[ ! -s "$GRANTSDESTFILE" ]];
 	then
@@ -75,45 +76,69 @@ function dump_grants
 
 }
 
-function mysqldump
+function dodump
 {
-	DUMPDEST="$DESTINATION/$BACKUPTS"
+	MYSQLDUMP_BASEOPTS=${MYSQLDUMP_BASEOPTS-"--opt --routines -E --master-data=$MASTERDATA"}
 
-	mkdir -p $DUMPDEST
+	CURRENTBACKUPLOGDUMPERR="${DUMPDESTFILE}.err"
 
-	DBS=${DBS-$(echo show databases | $MYSQLBIN --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf -N  | grep -vE '^(information|performance)_schema$|^mysql$')}
+	"$MYSQLDUMPBIN" $MYSQLDUMP_BASEOPTS $MYSQLDUMP_EXTRAOPTS --databases $DBS > $DUMPDESTFILE 2> ${CURRENTBACKUPLOGDUMPERR}
 
-	MASTERDATA=${MASTERDATA-1}
+	if [ "$?" -ne 0 ];
+	then
+		echo "mysqldump error, check logs"
+		BCKFAILED=1
+	fi
 
-	DUMPDESTFILE="$DUMPDEST/${IDHOST}.all.databases.sql"
+	if [ ! -z "$(cat ${CURRENTBACKUPLOGDUMPERR})" ];
+	then
+		echo "mysqldump error, check log ${CURRENTBACKUPLOGDUMPERR}"
+		BCKFAILED=1
+	fi
 
-	if [ -z "$DBS" ];
+	if [[ ! -s "$DUMPDESTFILE" ]];
 	then
-		echo "no dbs found"
+		echo "dump empty or not found, check logs"
+		BCKFAILED=1
+	fi
+}
+
+function mysqldump
+{
+	MYSQL_VER=$(echo "select version()" | $MYSQLBIN ${MYSQL_INSTANCE_OPTS} -NB 2>/dev/null)
+
+	if [ $? -ne 0 ];
+	then
+		echo "ERROR - mysql not available"
 		BCKFAILED=1
 	else
-		MYSQLDUMP_BASEOPTS=${MYSQLDUMP_BASEOPTS-"--opt --routines -E --master-data=$MASTERDATA"}
+		echo "MySQL ${MYSQL_VER}"
+	fi
 
-		CURRENTBACKUPLOGDUMPERR=${CURRENTBACKUPLOG-$DUMPDESTFILE.err}
+	DUMPDEST="$DESTINATION/$BACKUPTS"
 
-		"$MYSQLDUMPBIN" $MYSQLDUMP_BASEOPTS $MYSQLDUMP_EXTRAOPTS --databases $DBS > $DUMPDESTFILE 2> ${CURRENTBACKUPLOGDUMPERR}
+	mkdir -p $DUMPDEST
 
-		if [ "$?" -ne 0 ];
-		then
-			echo "mysqldump error, check logs"
-			BCKFAILED=1
-		fi
+	DBS=${DBS-$(echo show databases | $MYSQLBIN ${MYSQL_INSTANCE_OPTS} -N  | grep -vE '^(information|performance)_schema$|^mysql$|^sys$')}
 
-		if [ ! -z "$(cat ${CURRENTBACKUPLOG}.err)" ];
-		then
-			echo "mysqldump error, check log ${CURRENTBACKUPLOGDUMPERR}"
-			BCKFAILED=1
-		fi
+	MASTERDATA=${MASTERDATA-1}
 
-		if [[ ! -s "$DUMPDESTFILE" ]];
+	if [ -z "$DBS" ];
+	then
+		echo "no dbs found"
+		BCKFAILED=1
+	else
+		if [ -z "${FILE_PER_DB}" ];
 		then
-			echo "dump empty or not found, check logs"
-			BCKFAILED=1
+			DUMPDESTFILE="$DUMPDEST/${IDHOST}.all.databases.sql"
+			dodump
+		else
+			for EACHDB in $DBS;
+			do
+				EACHDB_FILE=$(echo "${EACHDB}" | sed 's/[^a-z0-9]/_/ig')
+				DUMPDESTFILE="$DUMPDEST/${IDHOST}.${EACHDB_FILE}.sql"
+				dodump
+			done
 		fi
 	fi
 
@@ -146,6 +171,11 @@ BASEDIRBCK=$(dirname $0)
 BASENAMEBCK=$(basename $0)
 IDHOST=${IDHOST-$(hostname -s)}
 
+if [ ! -z "${INSTANCE_NAME}" ];
+then
+	MYSQL_INSTANCE_OPTS="--defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf"
+fi
+
 if [ ! -z "$1" ] && [ -f "$1" ];
 then
 	. $1 2>/dev/null
@@ -173,7 +203,7 @@ then
 	BCKFAILED=1
 fi
 
-VERSIOMYSQL=$(echo 'select version();' | $MYSQLBIN --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf -N)
+VERSIOMYSQL=$(echo 'select version();' | $MYSQLBIN ${MYSQL_INSTANCE_OPTS} -N)
 
 if [ "$?" -ne 0 ];
 then

files/backup/mysqldump/backupmysqldumpconfig.template

@@ -0,0 +1,18 @@
+# backup destiation - mandatory
+DESTINATION="/backup"
+# notification email - optional
+MAILTO="[email protected]"
+# backup rotation in days - optional
+RETENTION="7"
+# log dir - stdout if unset
+LOGDIR="/backup/logs"
+# host id - hostname is used instead if it is unset
+IDHOST="ABC1234"
+# if set, enables compression
+COMPRESS="1"
+# mysql instance name - optional
+INSTANCE_NAME="<%= @instance %>"
+# masterdata - if not set, is set to 1
+MASTERDATA="2"
+# dump one db at a time if set - optional
+FILE_PER_DB="1"

templates/backup/xtrabackup/backupxtrabackup.erb → files/backup/xtrabackup/backupxtrabackup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#TODO: custom conf
+# puppet managed file
 
 function initbck
 {
@@ -68,8 +68,6 @@ function dobackup
 
 	#aqui logica fulls/diferencials:
 
-	#FULL_ON_MONTHDAY="<%= @fullbackup_monthday %>"
-	#FULL_ON_WEEKDAY="<%= @fullbackup_weekday %>"
 	if [ ! -z "$FULL_ON_MONTHDAY" ] && [ ! -z "$FULL_ON_WEEKDAY" ];
 	then
 		echo "FULL_ON_MONTHDAY and FULL_ON_WEEKDAY cannot be both defined"
@@ -109,7 +107,7 @@ function dobackup
 	if [ "$TODAY_IS_FULL" -eq 1 ];
 	then
 		# full
-		innobackupex --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf ${DUMPDEST}
+		innobackupex ${MYSQL_INSTANCE_OPTS} ${DUMPDEST}
 	else
 		# incremental
 
@@ -118,7 +116,7 @@ function dobackup
 		# innobackupex --incremental /var/backups/xtrabackup/ --incremental-basedir=/var/backups/xtrabackup/2014-08-25_10-04-45/
 
 		# de moment full :D
-		innobackupex --defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf ${DUMPDEST}
+		innobackupex ${MYSQL_INSTANCE_OPTS} ${DUMPDEST}
 	fi
 
 	if [ "$?" -ne 0 ];
@@ -153,6 +151,11 @@ BASENAMEBCK=$(basename $0)
 IDHOST=${IDHOST-$(hostname -s)}
 BACKUP_NAME_ID=${BACKUP_NAME_ID-MySQL}
 
+if [ ! -z "${INSTANCE_NAME}" ];
+then
+	MYSQL_INSTANCE_OPTS="--defaults-file=/etc/mysql/${INSTANCE_NAME}/my.cnf"
+fi
+
 if [ ! -z "$1" ] && [ -f "$1" ];
 then
 	. $1 2>/dev/null
@@ -188,5 +191,5 @@ then
 	date
 fi
 
-mailer
 cleanup
+mailer

files/backup/xtrabackup/backupxtrabackupconfig.template

@@ -0,0 +1,14 @@
+# backup destiation - mandatory
+DESTINATION="/backup"
+# notification email - optional
+MAILTO="[email protected]"
+# backup rotation in days - optional
+RETENTION="7"
+# log dir - stdout if unset
+LOGDIR="/backup/logs"
+# host id - hostname is used instead if it is unset
+IDHOST="ABC1234"
+# backup type
+BACKUP_NAME_ID="MySQL"
+# mysql instance name - optional
+INSTANCE_NAME="<%= @instance %>"

lib/puppet/provider/mysql_sql/mysql.rb

@@ -18,6 +18,7 @@ def run_sql_command(sql)
     command.push("--defaults-group-suffix=" + resource[:instance_name]) if resource[:instance_name]
     command.push("-S", resource[:socket]) if resource[:socket]
     command.push("-p" + resource[:password]) if resource[:password]
+    command.push("-BN")
     command.push("-e", '"' + sql.gsub('"', '\"') + '"')
     command.push(resource[:db]) if resource[:db]
 

lib/puppet/type/mysql_sql.rb

@@ -66,7 +66,7 @@ def sync
     # Return true if a matching row is found
     def matches(value)
       output, status = provider.run_unless_sql_command(value)
-      self.fail("Error evaluating 'unless' clause, returned #{status}: '#{output}'") unless status == 0
+      output='0' unless status == 0
 
       result_count = output.strip.to_i
       self.debug("Found #{result_count} row(s) executing 'unless' clause")
@@ -80,16 +80,16 @@ def matches(value)
         "for the existence of an object in the database to determine whether " +
         "or not the main SQL command needs to be executed at all."
 
-    # Return true if a matching row is found
+    # Return true if no matching row is found
     def matches(value)
       output, status = provider.run_unless_sql_command(value)
       status = output.exitcode if status.nil?
 
-      self.fail("Error evaluating 'onlyif' clause, returned #{status}: '#{output}'") unless status == 0
+      output='0' unless status == 0
 
       result_count = output.strip.to_i
       self.debug("Found #{result_count} row(s) executing 'onlyif' clause")
-      result_count > 0
+      result_count == 0
     end
   end
 

manifests/backup/mysqldump.pp

@@ -13,6 +13,8 @@
                                   $monthday     = undef,
                                   $weekday      = undef,
                                   $setcron      = true,
+                                  $masterdata   = '1',
+                                  $file_per_db  = true,
                                 ) {
   #
   validate_absolute_path($destination)
@@ -40,7 +42,7 @@
     owner   => 'root',
     group   => 'root',
     mode    => '0700',
-    content => template("${module_name}/backup/mysqldump/backupmysqldump.erb")
+    content => file("${module_name}/backup/mysqldump/backupmysqldump.sh")
   }
 
   file { "${backupscript}.config":

manifests/backup/xtrabackup.pp

@@ -50,7 +50,7 @@
     owner   => 'root',
     group   => 'root',
     mode    => '0700',
-    content => template("${module_name}/backup/xtrabackup/backupxtrabackup.erb")
+    content => file("${module_name}/backup/xtrabackup/backupxtrabackup.sh")
   }
 
   file { "${backupscript}.config":

manifests/mycnf/mysqld.pp

@@ -7,6 +7,7 @@
                               $port                            = '3306',
                               $pidfile                         = undef,
                               $datadir                         = "/var/mysql/${name}",
+                              $instancedir                     = "/var/mysql/${name}",
                               $relaylogdir                     = "/var/mysql/${name}/relaylogs",
                               $binlogdir                       = "/var/mysql/${name}/binlogs",
                               $default_storage_engine          = 'InnoDB',
@@ -52,6 +53,8 @@
                               $slow_query_log                  = true,
                               $log_error                       = "/var/log/mysql/${name}/mysql-error.log",
                               $slow_query_log_file             = "/var/log/mysql/${name}/mysql-slow.log",
+                              $sql_mode                        = undef,
+                              $lower_case_table_names          = '0',
                             ) {
   if($instance_name=='global')
   {

manifests/tools/perconatoolkit.pp

@@ -3,6 +3,13 @@
                                   ) inherits mysql::params {
   include ::mysql::perconarepo
 
+  # nota compatibilitat
+  #
+  # Error Summary
+  # -------------
+  # Error: /Stage[main]/Mysql::Tools::Perconatoolkit/Package[percona-toolkit]/ensure: change from purged to present failed: Execution of '/usr/bin/yum -d 0 -e 0 -y install percona-toolkit' returned 1: Transaction Check Error:
+  #   file /etc/my.cnf from install of Percona-Server-shared-51-5.1.73-rel14.12.625.rhel6.x86_64 conflicts with file from package mysql-community-server-5.7.20-1.el6.x86_64
+
   package { 'percona-toolkit':
     ensure  => $package_ensure,
     require => Class['::mysql::perconarepo'],

metadata.json

@@ -1,15 +1,15 @@
 {
   "name": "eyp-mysql",
-  "version": "0.4.11",
+  "version": "0.4.13",
   "author": "eyp",
   "summary": "multi instance mysql community and Percona XtraDB cluster (galera)",
   "license": "Apache-2.0",
   "source": "https://github.com/NTTCom-MS/eyp-mysql",
-  "project_page": null,
+  "project_page": "https://github.com/NTTCom-MS/eyp-mysql",
   "issues_url": "https://github.com/NTTCom-MS/eyp-mysql/issues",
   "dependencies": [
-    {"name":"puppetlabs/stdlib","version_requirement":">= 1.0.0"},
-    {"name":"puppetlabs/concat","version_requirement":">= 1.2.3"},
+    {"name":"puppetlabs/stdlib","version_requirement":">= 1.0.0 < 9.9.9"},
+    {"name":"puppetlabs/concat","version_requirement":">= 1.2.3 < 9.9.9"},
     {"name":"eyp/eyplib","version_requirement":">= 0.1.0 < 0.2.0"},
     {"name":"eyp/systemd","version_requirement":">= 0.1.25 < 0.2.0"},
     {"name":"eyp/initscript","version_requirement":">= 0.1.2 < 0.2.0"}