Store a MD5 hash of content for stored messages

alembic/versions/a192a8d3282c_add_content_hash.py

@@ -0,0 +1,27 @@
+"""
+Add content_hash field to messages model.
+
+Revision ID: a192a8d3282c
+Revises: 01b101590e74
+Create Date: 2024-09-10 16:32:46.593911
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a192a8d3282c"
+down_revision = "01b101590e74"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Apply the current migration."""
+    op.add_column("messages", sa.Column("content_hash", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    """Revert the current migration."""
+    op.drop_column("messages", "content_hash")

metricity/__main__.py

@@ -24,6 +24,7 @@ async def main() -> None:
         voice_states=False,
         presences=False,
         messages=True,
+        message_content=True,
         reactions=False,
         typing=False,
     )

metricity/exts/event_listeners/_syncer_utils.py

@@ -1,3 +1,6 @@
+import binascii
+import hashlib
+
 import discord
 from pydis_core.utils import logging
 from sqlalchemy import update
@@ -30,11 +33,17 @@ async def sync_message(message: discord.Message, sess: AsyncSession, *, from_thr
     if await sess.get(models.Message, str(message.id)):
         return
 
+    hash_ctx = hashlib.md5()  # noqa: S324
+    hash_ctx.update(message.content.encode())
+    digest = hash_ctx.digest()
+    digest_encoded = binascii.hexlify(digest).decode()
+
     args = {
         "id": str(message.id),
         "channel_id": str(message.channel.id),
         "author_id": str(message.author.id),
         "created_at": message.created_at,
+        "content_hash": digest_encoded,
     }
 
     if from_thread:

metricity/models.py

@@ -78,3 +78,4 @@ class Message(Base):
     author_id: Mapped[str] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
     created_at = mapped_column(TZDateTime())
     is_deleted: Mapped[bool] = mapped_column(default=False)
+    content_hash: Mapped[str] = mapped_column(nullable=True)