diff --git a/application/Module.php b/application/Module.php index 13723e504..e80a5b1f1 100644 --- a/application/Module.php +++ b/application/Module.php @@ -710,7 +710,8 @@ public function searchFulltext(ZendEvent $event) } $qb = $event->getParam('queryBuilder'); - $match = 'MATCH(omeka_fulltext_search.title, omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search)'; + $match = '(MATCH(omeka_fulltext_search.title, omeka_fulltext_search.record) AGAINST (:omeka_fulltext_search) > 0 OR MATCH(omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search) > 0)'; + $matchOrder = '(MATCH(omeka_fulltext_search.title, omeka_fulltext_search.record) AGAINST (:omeka_fulltext_search) OR MATCH(omeka_fulltext_search.text) AGAINST (:omeka_fulltext_search))'; if ('api.search.query' === $event->getName()) { @@ -727,7 +728,7 @@ public function searchFulltext(ZendEvent $event) $qb->innerJoin('Omeka\Entity\FulltextSearch', 'omeka_fulltext_search', 'WITH', $joinConditions); // Filter out resources with no similarity. - $qb->andWhere(sprintf('%s > 0', $match)); + $qb->andWhere($match); // Set visibility constraints. $acl = $this->getServiceLocator()->get('Omeka\Acl'); @@ -754,7 +755,7 @@ public function searchFulltext(ZendEvent $event) if (isset($query['sort_by_default']) || !$qb->getDQLPart('orderBy')) { $sortOrder = 'asc' === $query['sort_order'] ? 'ASC' : 'DESC'; - $qb->orderBy($match, $sortOrder); + $qb->orderBy($matchOrder, $sortOrder); } } } diff --git a/application/data/doctrine-proxies/__CG__OmekaEntityFulltextSearch.php b/application/data/doctrine-proxies/__CG__OmekaEntityFulltextSearch.php index 5e8bd774a..fa289b157 100644 --- a/application/data/doctrine-proxies/__CG__OmekaEntityFulltextSearch.php +++ b/application/data/doctrine-proxies/__CG__OmekaEntityFulltextSearch.php @@ -67,10 +67,10 @@ public function __construct(?\Closure $initializer = null, ?\Closure $cloner = n public function __sleep() { if ($this->__isInitialized__) { - return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'text']; + return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'record', 'text']; } - return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'text']; + return ['__isInitialized__', 'id', 'resource', 'owner', 'isPublic', 'title', 'record', 'text']; } /** @@ -273,6 +273,28 @@ public function getTitle() return parent::getTitle(); } + /** + * {@inheritDoc} + */ + public function setRecord($record) + { + + $this->__initializer__ && $this->__initializer__->__invoke($this, 'setRecord', [$record]); + + return parent::setRecord($record); + } + + /** + * {@inheritDoc} + */ + public function getRecord() + { + + $this->__initializer__ && $this->__initializer__->__invoke($this, 'getRecord', []); + + return parent::getRecord(); + } + /** * {@inheritDoc} */ diff --git a/application/data/install/schema.sql b/application/data/install/schema.sql index 5aeaf9fe0..6a526b317 100644 --- a/application/data/install/schema.sql +++ b/application/data/install/schema.sql @@ -30,10 +30,13 @@ CREATE TABLE `fulltext_search` ( `owner_id` int DEFAULT NULL, `is_public` tinyint(1) NOT NULL, `title` longtext COLLATE utf8mb4_unicode_ci, + `record` longtext COLLATE utf8mb4_unicode_ci, `text` longtext COLLATE utf8mb4_unicode_ci, PRIMARY KEY (`id`,`resource`), KEY `IDX_AA31FE4A7E3C61F9` (`owner_id`), - FULLTEXT KEY `IDX_AA31FE4A2B36786B3B8BA7C7` (`title`,`text`), + KEY `is_public` (`is_public`), + FULLTEXT KEY `IDX_AA31FE4A2B36786B9B349F91` (`title`,`record`), + FULLTEXT KEY `IDX_AA31FE4A3B8BA7C7` (`text`), CONSTRAINT `FK_AA31FE4A7E3C61F9` FOREIGN KEY (`owner_id`) REFERENCES `user` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; CREATE TABLE `item` ( diff --git a/application/data/migrations/20240219000003_AddIndexFullTextIsPublic.php b/application/data/migrations/20240219000003_AddIndexFullTextIsPublic.php new file mode 100644 index 000000000..8f445bf46 --- /dev/null +++ b/application/data/migrations/20240219000003_AddIndexFullTextIsPublic.php @@ -0,0 +1,21 @@ +executeStatement($sql); + } catch (\Exception $e) { + // Index exists. + } + } +} diff --git a/application/data/migrations/20240219000004_SeparateRecordAndTextForFullText.php b/application/data/migrations/20240219000004_SeparateRecordAndTextForFullText.php new file mode 100644 index 000000000..a9d055d7e --- /dev/null +++ b/application/data/migrations/20240219000004_SeparateRecordAndTextForFullText.php @@ -0,0 +1,49 @@ +jobDispatcher = $jobDispatcher; + } + + public function up(Connection $conn) + { + $sql = <<<'SQL' +TRUNCATE TABLE `fulltext_search`; + +ALTER TABLE `fulltext_search` +ADD `record` longtext COLLATE 'utf8mb4_unicode_ci' NULL AFTER `title`; + +ALTER TABLE `fulltext_search` +DROP INDEX `IDX_AA31FE4A2B36786B3B8BA7C7`; + +ALTER TABLE `fulltext_search` +ADD FULLTEXT `IDX_AA31FE4A2B36786B9B349F91` (`title`, `record`); + +ALTER TABLE `fulltext_search` +ADD FULLTEXT `IDX_AA31FE4A3B8BA7C7` (`text`); + +SQL; + $conn->executeStatement($sql); + + $this->jobDispatcher->dispatch(\DerivativeMedia\Job\DerivativeItem::class); + } + + public static function create(ServiceLocatorInterface $services) + { + return new self($services->get(\Omeka\Job\Dispatcher::class)); + } +} diff --git a/application/src/Api/Adapter/AbstractResourceEntityAdapter.php b/application/src/Api/Adapter/AbstractResourceEntityAdapter.php index 12d3c2661..c9b591b3a 100644 --- a/application/src/Api/Adapter/AbstractResourceEntityAdapter.php +++ b/application/src/Api/Adapter/AbstractResourceEntityAdapter.php @@ -715,7 +715,17 @@ public function getFulltextTitle($resource) return $resource->getTitle(); } + public function getFulltextRecord($resource) + { + return $this->getFulltext($resource, 'record'); + } + public function getFulltextText($resource) + { + return $this->getFulltext($resource, 'text'); + } + + protected function getFulltext($resource, string $type) { $services = $this->getServiceLocator(); $dataTypes = $services->get('Omeka\DataTypeManager'); @@ -723,7 +733,11 @@ public function getFulltextText($resource) $eventManager = $this->getEventManager(); $criteria = Criteria::create()->where(Criteria::expr()->eq('isPublic', true)); - $args = $eventManager->prepareArgs(['resource' => $resource, 'criteria' => $criteria]); + $args = $eventManager->prepareArgs([ + 'resource' => $resource, + 'type' => $type, + 'criteria' => $criteria, + ]); $event = new Event('api.get_fulltext_text.value_criteria', $this, $args); $eventManager->triggerEvent($event); $criteria = $args['criteria']; @@ -738,6 +752,7 @@ public function getFulltextText($resource) $valueAnnotationCriteria = Criteria::create()->where(Criteria::expr()->eq('isPublic', true)); $args = $eventManager->prepareArgs([ 'resource' => $resource, + 'type' => $type, 'value' => $value, 'criteria' => $valueAnnotationCriteria, ]); diff --git a/application/src/Api/Adapter/FulltextSearchableInterface.php b/application/src/Api/Adapter/FulltextSearchableInterface.php index ecdd45082..ddf339754 100644 --- a/application/src/Api/Adapter/FulltextSearchableInterface.php +++ b/application/src/Api/Adapter/FulltextSearchableInterface.php @@ -28,7 +28,15 @@ public function getFulltextIsPublic($resource); public function getFulltextTitle($resource); /** - * Get the the text of the passed resource. + * Get the record of the passed resource. + * + * @param mixed $resource + * @return string + */ + public function getFulltextRecord($resource); + + /** + * Get the the raw text (transcription, ocr, etc.) of the passed resource. * * @param mixed $resource * @return string diff --git a/application/src/Api/Adapter/ItemAdapter.php b/application/src/Api/Adapter/ItemAdapter.php index d85bebd43..a9074cf0c 100644 --- a/application/src/Api/Adapter/ItemAdapter.php +++ b/application/src/Api/Adapter/ItemAdapter.php @@ -328,6 +328,22 @@ public function preprocessBatchUpdate(array $data, Request $request) return $data; } + public function getFulltextRecord($resource) + { + $texts = []; + $texts[] = parent::getFulltextRecord($resource); + // Get media text. + $mediaAdapter = $this->getAdapter('media'); + foreach ($resource->getMedia() as $media) { + $texts[] = $mediaAdapter->getFulltextRecord($media); + } + // Remove empty texts. + $texts = array_filter($texts, function ($text) { + return !is_null($text) && $text !== ''; + }); + return implode("\n", $texts); + } + public function getFulltextText($resource) { $texts = []; diff --git a/application/src/Api/Adapter/MediaAdapter.php b/application/src/Api/Adapter/MediaAdapter.php index d16f4e5a2..9823a3226 100644 --- a/application/src/Api/Adapter/MediaAdapter.php +++ b/application/src/Api/Adapter/MediaAdapter.php @@ -202,6 +202,18 @@ public function preprocessBatchUpdate(array $data, Request $request) return $data; } + public function getFulltextRecord($resource) + { + $renderer = $this->getServiceLocator() + ->get('Omeka\Media\Renderer\Manager') + ->get($resource->getRenderer()); + $fulltextRecord = parent::getFulltextRecord($resource); + if ($renderer instanceof FulltextSearchableInterface) { + $fulltextRecord .= ' ' . $renderer->getFulltextRecord($this->getRepresentation($resource)); + } + return $fulltextRecord; + } + public function getFulltextText($resource) { $renderer = $this->getServiceLocator() diff --git a/application/src/Api/Adapter/SitePageAdapter.php b/application/src/Api/Adapter/SitePageAdapter.php index 5a4fb6c83..c6365e4c0 100644 --- a/application/src/Api/Adapter/SitePageAdapter.php +++ b/application/src/Api/Adapter/SitePageAdapter.php @@ -355,6 +355,11 @@ public function getFulltextTitle($resource) return $resource->getTitle(); } + public function getFulltextRecord($resource) + { + return ''; + } + public function getFulltextText($resource) { $services = $this->getServiceLocator(); diff --git a/application/src/Entity/FulltextSearch.php b/application/src/Entity/FulltextSearch.php index 469db6222..dc3bcfafc 100644 --- a/application/src/Entity/FulltextSearch.php +++ b/application/src/Entity/FulltextSearch.php @@ -5,7 +5,9 @@ * @Entity * @Table( * indexes={ - * @Index(columns={"title", "text"}, flags={"fulltext"}) + * @Index(name="is_public", columns={"is_public"}), + * @Index(columns={"title", "record"}, flags={"fulltext"}), + * @Index(columns={"text"}, flags={"fulltext"}) * } * ) */ @@ -39,6 +41,11 @@ class FulltextSearch */ protected $title; + /** + * @Column(type="text", nullable=true) + */ + protected $record; + /** * @Column(type="text", nullable=true) */ @@ -94,6 +101,16 @@ public function getTitle() return $this->title; } + public function setRecord($record) + { + $this->record = $record; + } + + public function getRecord() + { + return $this->record; + } + public function setText($text) { $this->text = $text; diff --git a/application/src/Job/IndexFulltextSearch.php b/application/src/Job/IndexFulltextSearch.php index 8b1dd235e..8d0b13be1 100644 --- a/application/src/Job/IndexFulltextSearch.php +++ b/application/src/Job/IndexFulltextSearch.php @@ -21,7 +21,7 @@ public function perform() // First delete all rows from the fulltext table to clear out the // resources that don't belong. - $conn->executeStatement('DELETE FROM `fulltext_search`'); + $conn->executeStatement('TRUNCATE TABLE `fulltext_search`'); // Then iterate through all resource types and index the ones that are // fulltext searchable. Note that we don't index "resource" and "value diff --git a/application/src/Media/Renderer/FulltextSearchableInterface.php b/application/src/Media/Renderer/FulltextSearchableInterface.php index 5b0d5a10c..003da923c 100644 --- a/application/src/Media/Renderer/FulltextSearchableInterface.php +++ b/application/src/Media/Renderer/FulltextSearchableInterface.php @@ -8,7 +8,7 @@ interface FulltextSearchableInterface /** * Get the the text of the passed media. * - * @param Media $media + * @param MediaRepresentation $media * @return string */ public function getFulltextText(MediaRepresentation $media); diff --git a/application/src/Stdlib/FulltextSearch.php b/application/src/Stdlib/FulltextSearch.php index 8ec27a56f..278da527f 100644 --- a/application/src/Stdlib/FulltextSearch.php +++ b/application/src/Stdlib/FulltextSearch.php @@ -32,11 +32,11 @@ public function save(ResourceInterface $resource, AdapterInterface $adapter) $ownerId = $owner ? $owner->getId() : null; $sql = 'INSERT INTO `fulltext_search` ( - `id`, `resource`, `owner_id`, `is_public`, `title`, `text` + `id`, `resource`, `owner_id`, `is_public`, `title`, `record`, `text` ) VALUES ( - :id, :resource, :owner_id, :is_public, :title, :text + :id, :resource, :owner_id, :is_public, :title, :record, :text ) ON DUPLICATE KEY UPDATE - `owner_id` = :owner_id, `is_public` = :is_public, `title` = :title, `text` = :text'; + `owner_id` = :owner_id, `is_public` = :is_public, `title` = :title, `record` = :record, `text` = :text'; $stmt = $this->conn->prepare($sql); $stmt->bindValue('id', $resourceId, PDO::PARAM_INT); @@ -44,6 +44,7 @@ public function save(ResourceInterface $resource, AdapterInterface $adapter) $stmt->bindValue('owner_id', $ownerId, PDO::PARAM_INT); $stmt->bindValue('is_public', $adapter->getFulltextIsPublic($resource), PDO::PARAM_BOOL); $stmt->bindValue('title', $adapter->getFulltextTitle($resource), PDO::PARAM_STR); + $stmt->bindValue('record', $adapter->getFulltextRecord($resource), PDO::PARAM_STR); $stmt->bindValue('text', $adapter->getFulltextText($resource), PDO::PARAM_STR); $stmt->executeStatement(); }