Separate record and text in full text search

application/Module.php

@@ -710,7 +710,8 @@ public function searchFulltext(ZendEvent $event)
         }
         $qb = $event->getParam('queryBuilder');
 
-        $match = 'MATCH(omeka_fulltext_search.title, omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search)';
+        $match = '(MATCH(omeka_fulltext_search.title, omeka_fulltext_search.record) AGAINST (:omeka_fulltext_search) > 0 OR MATCH(omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search) > 0)';
+        $matchOrder = '(MATCH(omeka_fulltext_search.title, omeka_fulltext_search.record) AGAINST (:omeka_fulltext_search) OR MATCH(omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search))';
 
         if ('api.search.query' === $event->getName()) {
 
@@ -727,7 +728,7 @@ public function searchFulltext(ZendEvent $event)
             $qb->innerJoin('Omeka\Entity\FulltextSearch', 'omeka_fulltext_search', 'WITH', $joinConditions);
 
             // Filter out resources with no similarity.
-            $qb->andWhere(sprintf('%s > 0', $match));
+            $qb->andWhere($match);
 
             // Set visibility constraints.
             $acl = $this->getServiceLocator()->get('Omeka\Acl');
@@ -754,7 +755,7 @@ public function searchFulltext(ZendEvent $event)
 
             if (isset($query['sort_by_default']) || !$qb->getDQLPart('orderBy')) {
                 $sortOrder = 'asc' === $query['sort_order'] ? 'ASC' : 'DESC';
-                $qb->orderBy($match, $sortOrder);
+                $qb->orderBy($matchOrder, $sortOrder);
             }
         }
     }

application/data/doctrine-proxies/__CG__OmekaEntityFulltextSearch.php

@@ -67,10 +67,10 @@ public function __construct(?\Closure $initializer = null, ?\Closure $cloner = n
     public function __sleep()
     {
         if ($this->__isInitialized__) {
-            return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'text'];
+            return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'record', 'text'];
         }
 
-        return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'text'];
+        return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'record', 'text'];
     }
 
     /**
@@ -273,6 +273,28 @@ public function getTitle()
         return parent::getTitle();
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    public function setRecord($record)
+    {
+
+        $this->__initializer__ && $this->__initializer__->__invoke($this, 'setRecord', [$record]);
+
+        return parent::setRecord($record);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getRecord()
+    {
+
+        $this->__initializer__ && $this->__initializer__->__invoke($this, 'getRecord', []);
+
+        return parent::getRecord();
+    }
+
     /**
      * {@inheritDoc}
      */

application/data/install/schema.sql

@@ -30,10 +30,13 @@ CREATE TABLE `fulltext_search` (
   `owner_id` int DEFAULT NULL,
   `is_public` tinyint(1) NOT NULL,
   `title` longtext COLLATE utf8mb4_unicode_ci,
+  `record` longtext COLLATE utf8mb4_unicode_ci,
   `text` longtext COLLATE utf8mb4_unicode_ci,
   PRIMARY KEY (`id`,`resource`),
   KEY `IDX_AA31FE4A7E3C61F9` (`owner_id`),
-  FULLTEXT KEY `IDX_AA31FE4A2B36786B3B8BA7C7` (`title`,`text`),
+  KEY `is_public` (`is_public`),
+  FULLTEXT KEY `IDX_AA31FE4A2B36786B9B349F91` (`title`,`record`),
+  FULLTEXT KEY `IDX_AA31FE4A3B8BA7C7` (`text`),
   CONSTRAINT `FK_AA31FE4A7E3C61F9` FOREIGN KEY (`owner_id`) REFERENCES `user` (`id`) ON DELETE SET NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 CREATE TABLE `item` (

application/data/migrations/20240219000003_AddIndexFullTextIsPublic.php

@@ -0,0 +1,21 @@
+<?php declare(strict_types=1);
+
+namespace Omeka\Db\Migrations;
+
+use Doctrine\DBAL\Connection;
+use Omeka\Db\Migration\MigrationInterface;
+
+class AddIndexFullTextIsPublic implements MigrationInterface
+{
+    public function up(Connection $conn)
+    {
+        $sql = <<<'SQL'
+ALTER TABLE `fulltext_search` ADD INDEX `is_public` (`is_public`);
+SQL;
+        try {
+            $conn->executeStatement($sql);
+        } catch (\Exception $e) {
+            // Index exists.
+        }
+    }
+}

application/data/migrations/20240219000004_SeparateRecordAndTextForFullText.php

@@ -0,0 +1,49 @@
+<?php declare(strict_types=1);
+
+namespace Omeka\Db\Migrations;
+
+use Doctrine\DBAL\Connection;
+use Laminas\ServiceManager\ServiceLocatorInterface;
+use Omeka\Db\Migration\ConstructedMigrationInterface;
+use Omeka\Job\Dispatcher as JobDispatcher;
+
+class SeparateRecordAndTextForFullText implements ConstructedMigrationInterface
+{
+    /**
+     * @var \Omeka\Job\Dispatcher
+     */
+    private $jobDispatcher;
+
+    public function __construct(JobDispatcher $jobDispatcher)
+    {
+        $this->jobDispatcher = $jobDispatcher;
+    }
+
+    public function up(Connection $conn)
+    {
+        $sql = <<<'SQL'
+TRUNCATE TABLE `fulltext_search`;
+
+ALTER TABLE `fulltext_search`
+ADD `record` longtext COLLATE 'utf8mb4_unicode_ci' NULL AFTER `title`;
+
+ALTER TABLE `fulltext_search`
+DROP INDEX `IDX_AA31FE4A2B36786B3B8BA7C7`;
+
+ALTER TABLE `fulltext_search`
+ADD FULLTEXT `IDX_AA31FE4A2B36786B9B349F91` (`title`, `record`);
+
+ALTER TABLE `fulltext_search`
+ADD FULLTEXT `IDX_AA31FE4A3B8BA7C7` (`text`);
+
+SQL;
+        $conn->executeStatement($sql);
+
+        $this->jobDispatcher->dispatch(\DerivativeMedia\Job\DerivativeItem::class);
+    }
+
+    public static function create(ServiceLocatorInterface $services)
+    {
+        return new self($services->get(\Omeka\Job\Dispatcher::class));
+    }
+}

application/src/Api/Adapter/AbstractResourceEntityAdapter.php

@@ -715,15 +715,29 @@ public function getFulltextTitle($resource)
         return $resource->getTitle();
     }
 
+    public function getFulltextRecord($resource)
+    {
+        return $this->getFulltext($resource, 'record');
+    }
+
     public function getFulltextText($resource)
+    {
+        return $this->getFulltext($resource, 'text');
+    }
+
+    protected function getFulltext($resource, string $type)
     {
         $services = $this->getServiceLocator();
         $dataTypes = $services->get('Omeka\DataTypeManager');
         $view = $services->get('ViewRenderer');
         $eventManager = $this->getEventManager();
 
         $criteria = Criteria::create()->where(Criteria::expr()->eq('isPublic', true));
-        $args = $eventManager->prepareArgs(['resource' => $resource, 'criteria' => $criteria]);
+        $args = $eventManager->prepareArgs([
+            'resource' => $resource,
+            'type' => $type,
+            'criteria' => $criteria,
+        ]);
         $event = new Event('api.get_fulltext_text.value_criteria', $this, $args);
         $eventManager->triggerEvent($event);
         $criteria = $args['criteria'];
@@ -738,6 +752,7 @@ public function getFulltextText($resource)
                 $valueAnnotationCriteria = Criteria::create()->where(Criteria::expr()->eq('isPublic', true));
                 $args = $eventManager->prepareArgs([
                     'resource' => $resource,
+                    'type' => $type,
                     'value' => $value,
                     'criteria' => $valueAnnotationCriteria,
                 ]);

application/src/Api/Adapter/FulltextSearchableInterface.php

@@ -28,7 +28,15 @@ public function getFulltextIsPublic($resource);
     public function getFulltextTitle($resource);
 
     /**
-     * Get the the text of the passed resource.
+     * Get the record of the passed resource.
+     *
+     * @param mixed $resource
+     * @return string
+     */
+    public function getFulltextRecord($resource);
+
+    /**
+     * Get the the raw text (transcription, ocr, etc.) of the passed resource.
      *
      * @param mixed $resource
      * @return string

application/src/Api/Adapter/ItemAdapter.php

@@ -328,6 +328,22 @@ public function preprocessBatchUpdate(array $data, Request $request)
         return $data;
     }
 
+    public function getFulltextRecord($resource)
+    {
+        $texts = [];
+        $texts[] = parent::getFulltextRecord($resource);
+        // Get media text.
+        $mediaAdapter = $this->getAdapter('media');
+        foreach ($resource->getMedia() as $media) {
+            $texts[] = $mediaAdapter->getFulltextRecord($media);
+        }
+        // Remove empty texts.
+        $texts = array_filter($texts, function ($text) {
+            return !is_null($text) && $text !== '';
+        });
+        return implode("\n", $texts);
+    }
+
     public function getFulltextText($resource)
     {
         $texts = [];

application/src/Api/Adapter/MediaAdapter.php

@@ -202,6 +202,18 @@ public function preprocessBatchUpdate(array $data, Request $request)
         return $data;
     }
 
+    public function getFulltextRecord($resource)
+    {
+        $renderer = $this->getServiceLocator()
+            ->get('Omeka\Media\Renderer\Manager')
+            ->get($resource->getRenderer());
+        $fulltextRecord = parent::getFulltextRecord($resource);
+        if ($renderer instanceof FulltextSearchableInterface) {
+            $fulltextRecord .= ' ' . $renderer->getFulltextRecord($this->getRepresentation($resource));
+        }
+        return $fulltextRecord;
+    }
+
     public function getFulltextText($resource)
     {
         $renderer = $this->getServiceLocator()

application/src/Api/Adapter/SitePageAdapter.php

@@ -355,6 +355,11 @@ public function getFulltextTitle($resource)
         return $resource->getTitle();
     }
 
+    public function getFulltextRecord($resource)
+    {
+        return '';
+    }
+
     public function getFulltextText($resource)
     {
         $services = $this->getServiceLocator();

application/src/Entity/FulltextSearch.php

@@ -5,7 +5,9 @@
  * @Entity
  * @Table(
  *   indexes={
- *     @Index(columns={"title", "text"}, flags={"fulltext"})
+ *     @Index(name="is_public", columns={"is_public"}),
+ *     @Index(columns={"title", "record"}, flags={"fulltext"}),
+ *     @Index(columns={"text"}, flags={"fulltext"})
  *   }
  * )
  */
@@ -39,6 +41,11 @@ class FulltextSearch
      */
     protected $title;
 
+    /**
+     * @Column(type="text", nullable=true)
+     */
+    protected $record;
+
     /**
      * @Column(type="text", nullable=true)
      */
@@ -94,6 +101,16 @@ public function getTitle()
         return $this->title;
     }
 
+    public function setRecord($record)
+    {
+        $this->record = $record;
+    }
+
+    public function getRecord()
+    {
+        return $this->record;
+    }
+
     public function setText($text)
     {
         $this->text = $text;

application/src/Job/IndexFulltextSearch.php

@@ -21,7 +21,7 @@ public function perform()
 
         // First delete all rows from the fulltext table to clear out the
         // resources that don't belong.
-        $conn->executeStatement('DELETE FROM `fulltext_search`');
+        $conn->executeStatement('TRUNCATE TABLE `fulltext_search`');
 
         // Then iterate through all resource types and index the ones that are
         // fulltext searchable. Note that we don't index "resource" and "value

application/src/Media/Renderer/FulltextSearchableInterface.php

@@ -8,7 +8,7 @@ interface FulltextSearchableInterface
     /**
      * Get the the text of the passed media.
      *
-     * @param Media $media
+     * @param MediaRepresentation $media
      * @return string
      */
     public function getFulltextText(MediaRepresentation $media);

application/src/Stdlib/FulltextSearch.php

@@ -32,18 +32,19 @@ public function save(ResourceInterface $resource, AdapterInterface $adapter)
         $ownerId = $owner ? $owner->getId() : null;
 
         $sql = 'INSERT INTO `fulltext_search` (
-            `id`, `resource`, `owner_id`, `is_public`, `title`, `text`
+            `id`, `resource`, `owner_id`, `is_public`, `title`, `record`, `text`
         ) VALUES (
-            :id, :resource, :owner_id, :is_public, :title, :text
+            :id, :resource, :owner_id, :is_public, :title, :record, :text
         ) ON DUPLICATE KEY UPDATE
-            `owner_id` = :owner_id, `is_public` = :is_public, `title` = :title, `text` = :text';
+            `owner_id` = :owner_id, `is_public` = :is_public, `title` = :title, `record` = :record, `text` = :text';
         $stmt = $this->conn->prepare($sql);
 
         $stmt->bindValue('id', $resourceId, PDO::PARAM_INT);
         $stmt->bindValue('resource', $resourceName, PDO::PARAM_STR);
         $stmt->bindValue('owner_id', $ownerId, PDO::PARAM_INT);
         $stmt->bindValue('is_public', $adapter->getFulltextIsPublic($resource), PDO::PARAM_BOOL);
         $stmt->bindValue('title', $adapter->getFulltextTitle($resource), PDO::PARAM_STR);
+        $stmt->bindValue('record', $adapter->getFulltextRecord($resource), PDO::PARAM_STR);
         $stmt->bindValue('text', $adapter->getFulltextText($resource), PDO::PARAM_STR);
         $stmt->executeStatement();
     }