From 4ed50a61c76938ded9bac87d8cb5a0db2ac8841b Mon Sep 17 00:00:00 2001
From: Matteo Lodi <30625432+mlodic@users.noreply.github.com>
Date: Thu, 16 Nov 2023 15:31:55 +0100
Subject: [PATCH 1/4] deleting one document at a time for upload method of
 BIDocument

---
 certego_saas/ext/upload/elastic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/certego_saas/ext/upload/elastic.py b/certego_saas/ext/upload/elastic.py
index 5b09271..d03073f 100644
--- a/certego_saas/ext/upload/elastic.py
+++ b/certego_saas/ext/upload/elastic.py
@@ -51,7 +51,8 @@ def upload(
         logger.info(f"Uploading {docs.count()} documents")
         jsons = map(lambda x: x.to_bulk(), docs)
         success, errors = bulk(client, jsons, request_timeout=timeout)
-        docs.delete()
+        for doc in docs:
+            doc.delete()
         return success, errors
 
     def clean(self):

From 2e6813366687e9089fc6e3a76282b59fb6573b23 Mon Sep 17 00:00:00 2001
From: Matteo Lodi <30625432+mlodic@users.noreply.github.com>
Date: Thu, 16 Nov 2023 15:33:18 +0100
Subject: [PATCH 2/4] bump

---
 certego_saas/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/certego_saas/version.py b/certego_saas/version.py
index a10d59f..6852e09 100644
--- a/certego_saas/version.py
+++ b/certego_saas/version.py
@@ -1 +1 @@
-VERSION = "0.7.3"
+VERSION = "0.7.4"

From 2d368c5b491657f58e8e74730e781e94eefb9ec1 Mon Sep 17 00:00:00 2001
From: Matteo Lodi <30625432+mlodic@users.noreply.github.com>
Date: Thu, 16 Nov 2023 15:40:23 +0100
Subject: [PATCH 3/4] added indexes for BIDocument

---
 certego_saas/ext/upload/elastic.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/certego_saas/ext/upload/elastic.py b/certego_saas/ext/upload/elastic.py
index d03073f..ab00c0d 100644
--- a/certego_saas/ext/upload/elastic.py
+++ b/certego_saas/ext/upload/elastic.py
@@ -69,7 +69,7 @@ def __repr__(self):
     from mongoengine import fields as mongo_fields
 
 except ImportError:
-    from django.db.models import JSONField, Model
+    from django.db.models import Index, JSONField, Model
     from django.db.models import fields as django_fields
 
     class BIDocument(__BIDocumentInterface, Model):
@@ -79,6 +79,16 @@ class BIDocument(__BIDocumentInterface, Model):
         count = django_fields.PositiveIntegerField()
         kwargs = JSONField()
 
+        class Meta:
+            indexes = [
+                Index(
+                    fields=[
+                        "index",
+                        "time",
+                    ]
+                )
+            ]
+
 else:
 
     class BIDocument(__BIDocumentInterface, Document):
@@ -89,3 +99,4 @@ class BIDocument(__BIDocumentInterface, Document):
         category = mongo_fields.StringField(required=True)
         count = mongo_fields.IntField(required=True, min_value=0)
         kwargs = mongo_fields.DictField(required=False)
+        meta = {"indexes": ["index", "time"]}

From 72508c366e8dd8913dc54fe54e4151ab49f0f94f Mon Sep 17 00:00:00 2001
From: Matteo Lodi <30625432+mlodic@users.noreply.github.com>
Date: Thu, 16 Nov 2023 16:21:47 +0100
Subject: [PATCH 4/4] added iterator to optimize memory usage

---
 certego_saas/ext/upload/elastic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/certego_saas/ext/upload/elastic.py b/certego_saas/ext/upload/elastic.py
index ab00c0d..070cbd5 100644
--- a/certego_saas/ext/upload/elastic.py
+++ b/certego_saas/ext/upload/elastic.py
@@ -51,7 +51,7 @@ def upload(
         logger.info(f"Uploading {docs.count()} documents")
         jsons = map(lambda x: x.to_bulk(), docs)
         success, errors = bulk(client, jsons, request_timeout=timeout)
-        for doc in docs:
+        for doc in docs.iterator():
             doc.delete()
         return success, errors