DKAN-4291 add mysql empty row removal option

modules/datastore/modules/datastore_mysql_import/datastore_mysql_import.info.yml

@@ -3,6 +3,7 @@ description: Provides a MySQL Importer class.
 type: module
 core_version_requirement: ^10
 package: DKAN
+configure: datastore.mysql_import.settings
 dependencies:
   - dkan:common
   - dkan:datastore

modules/datastore/modules/datastore_mysql_import/datastore_mysql_import.links.menu.yml

@@ -0,0 +1,6 @@
+datastore_mysql_import.settings_form:
+  title: MySQl Import settings
+  description: Setting for the MySQL Importer.
+  parent: system.admin_dkan
+  route_name: datastore.mysql_import.settings
+  weight: 15

modules/datastore/modules/datastore_mysql_import/datastore_mysql_import.routing.yml

@@ -0,0 +1,7 @@
+datastore.mysql_import.settings:
+  path: '/admin/dkan/datastore/mysql_import'
+  defaults:
+    _title: 'MySQL Import Settings'
+    _form: 'Drupal\datastore_mysql_import\Form\DatastoreMysqlImportSettingsForm'
+  requirements:
+    _permission: 'administer site configuration'

modules/datastore/modules/datastore_mysql_import/src/Form/DatastoreMysqlImportSettingsForm.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace Drupal\datastore_mysql_import\Form;
+
+use Drupal\Core\Config\ConfigFactoryInterface;
+use Drupal\Core\Form\ConfigFormBase;
+use Drupal\Core\Form\FormStateInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Datastore MySQL Import settings form.
+ *
+ * @package Drupal\datastore\Form
+ * @codeCoverageIgnore
+ */
+class DatastoreMysqlImportSettingsForm extends ConfigFormBase {
+
+
+  /**
+   * Constructs form.
+   *
+   * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory
+   *   The factory for configuration objects.
+   */
+  public function __construct(ConfigFactoryInterface $config_factory) {
+    parent::__construct($config_factory);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container) {
+    return new static(
+      $container->get('config.factory'),
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getFormId() {
+    return 'datastore_mysql_import_settings_form';
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function getEditableConfigNames() {
+    return ['datastore.mysql_import.settings'];
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function buildForm(array $form, FormStateInterface $form_state) {
+    $config = $this->config('datastore.mysql_import.settings');
+    $form['remove_empty_rows'] = [
+      '#type' => 'checkbox',
+      '#title' => $this->t('Enable removal of empty rows in dataset.'),
+      '#description' => $this->t('Unlike the chunk harvester, which ignores empty rows in a CSV, the MySQL importer will import empty rows.'),
+      '#default_value' => $config->get('remove_empty_rows'),
+    ];
+    return parent::buildForm($form, $form_state);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function submitForm(array &$form, FormStateInterface $form_state) {
+    $this->config('datastore.mysql_import.settings')
+      ->set('remove_empty_rows', $form_state->getValue('remove_empty_rows'))
+      ->save();
+    parent::submitForm($form, $form_state);
+  }
+
+}

modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php

@@ -103,6 +103,7 @@ protected function runIt() {
     Database::setActiveConnection();
 
     $this->setStatus(Result::DONE);
+    $this->runOptionalPostImportCleanup($this->dataStorage->getTableName(), $spec);
     return NULL;
   }
 
@@ -128,7 +129,7 @@ protected function getColsFromFile(string $file_path, string $delimiter): array
     $f = fopen($file_path, 'r');
 
     // Ensure the file could be successfully opened.
-    if ($f === FALSE) {
+    if (!isset($f) || $f === FALSE) {
       throw new FileException(sprintf('Failed to open resource file "%s".', $file_path));
     }
 
@@ -141,9 +142,8 @@ protected function getColsFromFile(string $file_path, string $delimiter): array
 
     // Close the resource file, since it is no longer needed.
     fclose($f);
-    // Ensure the columns of the resource file were successfully read. $columns
-    // could be array, FALSE, or NULL.
-    if (!$columns) {
+    // Ensure the columns of the resource file were successfully read.
+    if (!isset($columns) || $columns === FALSE) {
       throw new FileException(sprintf('Failed to read columns from resource file "%s".', $file_path));
     }
 
@@ -257,4 +257,42 @@ protected function getSqlStatement(string $file_path, string $table_name, array
     ]);
   }
 
+  /**
+   * Performs optional cleanup steps on imported data.
+   *
+   * @param string $table_name
+   *  The name of the table to clean.
+   * @param array $spec
+   *   The sanitized headers for the imported data.
+   */
+  protected function runOptionalPostImportCleanup(string $table_name, array $spec): void {
+    // @todo this should use dependency injection, but this is not a service.
+    $config = \Drupal::config('datastore.mysql_import.settings');
+    if ($config->get('remove_empty_rows')) {
+      $this->removeEmptyRows($table_name, $spec);
+    }
+  }
+
+  /**
+   * Remove empty rows from the imported data.
+   *
+   * @param string $table_name
+   *  The name of the table to remove empty rows from.
+   * @param array $fields
+   *   The sanitized headers for the imported data.
+   */
+  protected function removeEmptyRows(string $table_name, array $fields): void {
+    $connection = \Drupal::database();
+    $query = $connection->delete($table_name);
+    foreach ($fields as $field) {
+      // Build query to look for records with all empty values for all fields.
+      $query->condition($field, '', '=');
+    }
+    $num_deleted = $query->execute();
+    if ($num_deleted > 0) {
+      // @todo We should log how many were deleted... no logger :(
+
+    }
+  }
+
 }