From cc59781af622e08f459529d53c07794d8c961d65 Mon Sep 17 00:00:00 2001
From: Michal Kotek <michal.kotek@notix.cz>
Date: Sun, 17 May 2020 15:17:45 +0200
Subject: [PATCH 01/14] getFindQuery support for $and condition enabling more
 same name & operator queries

---
 Resource.js | 176 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 108 insertions(+), 68 deletions(-)

diff --git a/Resource.js b/Resource.js
index 02f467b..76c807d 100644
--- a/Resource.js
+++ b/Resource.js
@@ -404,77 +404,117 @@ class Resource {
    * @returns {Object}
    */
   getFindQuery(req, options) {
-    const findQuery = {};
+    const finalQuery = {"$and": []};
+    let findQuery = { };
     options = options || this.options;
 
     // Get the filters and omit the limit, skip, select, sort and populate.
     const {limit, skip, select, sort, populate, ...filters} = req.query;
 
     // Iterate through each filter.
-    Object.entries(filters).forEach(([name, value]) => {
+    Object.entries(filters).forEach(([name, values]) => {
+
+      if ( !Array.isArray(values) ) {
+        values = [values];
+      }
+
       // Get the filter object.
       const filter = utils.zipObject(['name', 'selector'], name.split('__'));
 
-      // See if this parameter is defined in our model.
-      const param = this.model.schema.paths[filter.name.split('.')[0]];
-      if (param) {
-        // See if this selector is a regular expression.
-        if (filter.selector === 'regex') {
-          // Set the regular expression for the filter.
-          const parts = value.match(/\/?([^/]+)\/?([^/]+)?/);
-          let regex = null;
-          try {
-            regex = new RegExp(parts[1], (parts[2] || 'i'));
-          }
-          catch (err) {
-            debug.query(err);
-            regex = null;
-          }
-          if (regex) {
-            findQuery[filter.name] = regex;
-          }
-          return;
-        } // See if there is a selector.
-        else if (filter.selector) {
-          // Init the filter.
-          if (!Object.prototype.hasOwnProperty.call(findQuery, filter.name)) {
-            findQuery[filter.name] = {};
-          }
+      for(let value of values) {
+        if ( !utils.isEmpty(findQuery) ) {
+          finalQuery.$and.push(findQuery);
+          findQuery = { };
+        }
 
-          if (filter.selector === 'exists') {
-            value = ((value === 'true') || (value === '1')) ? true : value;
-            value = ((value === 'false') || (value === '0')) ? false : value;
-            value = !!value;
-          }
-          // Special case for in filter with multiple values.
-          else if (['in', 'nin'].includes(filter.selector)) {
-            value = Array.isArray(value) ? value : value.split(',');
-            value = value.map((item) => Resource.getQueryValue(filter.name, item, param, options, filter.selector));
+        // OR condition from reference
+        if ( name === "$or" ) {
+          findQuery = {
+            $or: values
+          };
+          break;
+        }
+        // AND condition from recursive reference
+        else if ( name === "$and" ) {
+          finalQuery.$and = values;
+          findQuery = { };
+          break;
+        }
+
+        // See if this parameter is defined in our model.
+        const param = this.model.schema.paths[filter.name.split('.')[0]];
+        if (param) {
+          // See if there is a selector.
+          if (filter.selector) {
+            // See if this selector is a regular expression.
+            if (filter.selector === 'regex') {
+              // Set the regular expression for the filter.
+              const parts = value.match(/\/?([^/]+)\/?([^/]+)?/);
+              let regex = null;
+              try {
+                regex = new RegExp(parts[1], (parts[2] || 'i'));
+              }
+              catch (err) {
+                debug.query(err);
+                regex = null;
+              }
+              if (regex) {
+                findQuery[filter.name] = regex;
+              }
+              continue;
+            } // See if there is a selector.
+            else if (filter.selector) {
+              // Init the filter.
+              if (!Object.prototype.hasOwnProperty.call(findQuery, filter.name)) {
+                findQuery[filter.name] = {};
+              }
+
+              if (filter.selector === 'exists') {
+                value = ((value === 'true') || (value === '1')) ? true : value;
+                value = ((value === 'false') || (value === '0')) ? false : value;
+                value = !!value;
+              }
+              // Special case for in filter with multiple values.
+              else if (['in', 'nin'].includes(filter.selector)) {
+                value = Array.isArray(value) ? value : value.split(',');
+                value = value.map((item) => Resource.getQueryValue(filter.name, item, param, options, filter.selector));
+              }
+              else {
+                // Set the selector for this filter name.
+                value = Resource.getQueryValue(filter.name, value, param, options, filter.selector);
+              }
+
+
+              findQuery[filter.name][`$${filter.selector}`] = value;
+              continue;
+            }
           }
           else {
-            // Set the selector for this filter name.
+            // Set the find query to this value.
             value = Resource.getQueryValue(filter.name, value, param, options, filter.selector);
+            findQuery[filter.name] = value;
+            continue;
           }
-
-          findQuery[filter.name][`$${filter.selector}`] = value;
-          return;
         }
-        else {
+
+        if (!options.queryFilter) {
           // Set the find query to this value.
-          value = Resource.getQueryValue(filter.name, value, param, options, filter.selector);
           findQuery[filter.name] = value;
-          return;
         }
       }
-
-      if (!options.queryFilter) {
-        // Set the find query to this value.
-        findQuery[filter.name] = value;
-      }
     });
 
+    if ( !utils.isEmpty(findQuery) ) {
+      finalQuery.$and.push(findQuery);
+      findQuery = { };
+    }
+
+    if ( finalQuery.$and.length <= 0 ) {
+      return { };
+    }
+
     // Return the findQuery.
-    return findQuery;
+    return finalQuery;
   }
 
   countQuery(query, pipeline) {
@@ -564,11 +604,11 @@ class Resource {
 
         // Get the default limit.
         const defaults = { limit: 10, skip: 0 };
-        let { limit, skip } = req.query
-        limit = parseInt(limit, 10)
-        limit = (isNaN(limit) || (limit < 0)) ? defaults.limit : limit
-        skip = parseInt(skip, 10)
-        skip = (isNaN(skip) || (skip < 0)) ? defaults.skip : skip
+        let { limit, skip } = req.query;
+        limit = parseInt(limit, 10);
+        limit = (isNaN(limit) || (limit < 0)) ? defaults.limit : limit;
+        skip = parseInt(skip, 10);
+        skip = (isNaN(skip) || (skip < 0)) ? defaults.skip : skip;
         const reqQuery = { limit, skip };
 
         // If a skip is provided, then set the range headers.
@@ -821,22 +861,22 @@ class Resource {
           res,
           item,
           () => {
-          const writeOptions = req.writeOptions || {};
-          item.save(writeOptions, (err, item) => {
-            if (err) {
-              debug.put(err);
-              return Resource.setResponse(res, { status: 400, error: err }, next);
-            }
+            const writeOptions = req.writeOptions || {};
+            item.save(writeOptions, (err, item) => {
+              if (err) {
+                debug.put(err);
+                return Resource.setResponse(res, { status: 400, error: err }, next);
+              }
 
-            return options.hooks.put.after.call(
-              this,
-              req,
-              res,
-              item,
-              Resource.setResponse.bind(Resource, res, { status: 200, item }, next)
-            );
+              return options.hooks.put.after.call(
+                this,
+                req,
+                res,
+                item,
+                Resource.setResponse.bind(Resource, res, { status: 200, item }, next)
+              );
+            });
           });
-        });
       });
     }, Resource.respond, options);
     return this;

From 99f0378e3c6d0adc94343e9b59407bbd11ff3655 Mon Sep 17 00:00:00 2001
From: Michal Kotek <michal.kotek@notix.cz>
Date: Sun, 17 May 2020 20:09:16 +0200
Subject: [PATCH 02/14] Fixing support for multiple select query params

---
 Resource.js  |  4 +++-
 test/test.js | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/Resource.js b/Resource.js
index 02f467b..007a285 100644
--- a/Resource.js
+++ b/Resource.js
@@ -340,8 +340,10 @@ class Resource {
       return req.query[name];
     }
     else {
+      const query = ( Array.isArray(req.query[name]) ? req.query[name].join(',') : req.query[name] );
+
       // Generate string of spaced unique keys
-      return [...new Set(req.query[name].match(/[^, ]+/g))].join(' ')
+      return [...new Set(query.match(/[^, ]+/g))].join(' ');
     }
   }
 
diff --git a/test/test.js b/test/test.js
index bc5594d..73181bd 100644
--- a/test/test.js
+++ b/test/test.js
@@ -1248,6 +1248,23 @@ function testSearch(testPath) {
       });
     }));
 
+  it('Should be able to select fields with multiple select queries', () => request(app)
+    .get(`${testPath}?limit=10&skip=10&select=title&select=age`)
+    .expect('Content-Type', /json/)
+    .expect('Content-Range', '10-19/26')
+    .expect(206)
+    .then((res) => {
+      const response = res.body;
+      assert.equal(response.length, 10);
+      let age = 10;
+      response.forEach((resource) => {
+        assert.equal(resource.title, `Test Age ${age}`);
+        assert.equal(resource.description, undefined);
+        assert.equal(resource.age, age);
+        age++;
+      });
+    }));
+
   it('Should be able to sort', () => request(app)
     .get(`${testPath}?select=age&sort=-age`)
     .expect('Content-Type', /json/)

From 34d343144c7148bab06c907c8cf6380c456d0ad1 Mon Sep 17 00:00:00 2001
From: Michal Kotek <michal.kotek@notix.cz>
Date: Mon, 18 May 2020 12:20:23 +0200
Subject: [PATCH 03/14] Wrong separator for GET query select

---
 Resource.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Resource.js b/Resource.js
index 007a285..eb6accb 100644
--- a/Resource.js
+++ b/Resource.js
@@ -694,7 +694,7 @@ class Resource {
                   if (item._id) {
                     newItem._id = item._id;
                   }
-                  select.split(',').map(key => {
+                  select.split(' ').map(key => {
                     key = key.trim();
                     if (item.hasOwnProperty(key)) {
                       newItem[key] = item[key];

From d776261073751c2efe81adc2168741c99c2f6b04 Mon Sep 17 00:00:00 2001
From: Michal Kotek <michal.kotek@notix.cz>
Date: Mon, 18 May 2020 19:19:24 +0200
Subject: [PATCH 04/14] getFindQuery support for $and condition tests

---
 test/test.js | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/test/test.js b/test/test.js
index 73181bd..c318e0c 100644
--- a/test/test.js
+++ b/test/test.js
@@ -1443,6 +1443,17 @@ function testSearch(testPath) {
       });
     }));
 
+  it('multiple lt search selector', () => request(app)
+    .get(`${testPath}?age__lt=5&age__lt=3`)
+    .expect('Content-Range', '0-2/3')
+    .then((res) => {
+      const response = res.body;
+      assert.equal(response.length, 3);
+      response.forEach((resource) => {
+        assert.ok(resource.age < 5 && resource.age < 3);
+      });
+    }));
+
   it('lte search selector', () => request(app)
     .get(`${testPath}?age__lte=5`)
     .expect('Content-Range', '0-5/6')
@@ -1454,6 +1465,17 @@ function testSearch(testPath) {
       });
     }));
 
+  it('multiple lte search selector', () => request(app)
+    .get(`${testPath}?age__lte=5&age__lte=10`)
+    .expect('Content-Range', '0-5/6')
+    .then((res) => {
+      const response = res.body;
+      assert.equal(response.length, 6);
+      response.forEach((resource) => {
+        assert.ok(resource.age <= 5 && resource.age <= 10);
+      });
+    }));
+
   it('gt search selector', () => request(app)
     .get(`${testPath}?age__gt=5`)
     .expect('Content-Range', '0-9/19')
@@ -1465,6 +1487,17 @@ function testSearch(testPath) {
       });
     }));
 
+  it('multiple gt search selector', () => request(app)
+    .get(`${testPath}?age__gt=5&age__gt=10`)
+    .expect('Content-Range', '0-9/14')
+    .then((res) => {
+      const response = res.body;
+      assert.equal(response.length, 10);
+      response.forEach((resource) => {
+        assert.ok(resource.age > 5 && resource.age > 10);
+      });
+    }));
+
   it('gte search selector', () => request(app)
     .get(`${testPath}?age__gte=5`)
     .expect('Content-Range', '0-9/20')
@@ -1476,6 +1509,17 @@ function testSearch(testPath) {
       });
     }));
 
+  it('multiple gte search selector', () => request(app)
+    .get(`${testPath}?age__gte=5&age__gte=10`)
+    .expect('Content-Range', '0-9/15')
+    .then((res) => {
+      const response = res.body;
+      assert.equal(response.length, 10);
+      response.forEach((resource) => {
+        assert.ok(resource.age >= 5 && resource.age >= 10);
+      });
+    }));
+
   it('regex search selector', () => request(app)
     .get(`${testPath}?title__regex=/.*Age [0-1]?[0-3]$/g`)
     .expect('Content-Range', '0-7/8')
@@ -1488,6 +1532,16 @@ function testSearch(testPath) {
       });
     }));
 
+  it('multiple regex search selector', () => request(app)
+    .get(`${testPath}?limit=1000&description__regex=/^Descr.*$/&description__regex=/1/`)
+    .expect('Content-Range', '0-11/12')
+    .then((res) => {
+      const response = res.body;
+      response.forEach((resource) => {
+        assert.ok(resource.description.startsWith('Descr') && resource.description.indexOf('1') !== -1);
+      });
+    }));
+
   it('regex search selector should be case insensitive', () => {
     const name = resourceNames[0].toString();
 

From 7d8af4dc93adeeceb842fcbdf095a3fd20e9d98e Mon Sep 17 00:00:00 2001
From: Michal <xamien@seznam.cz>
Date: Mon, 17 Aug 2020 23:19:59 +0200
Subject: [PATCH 05/14] validationForm query path

---
 Resource.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Resource.js b/Resource.js
index c452746..25a0667 100644
--- a/Resource.js
+++ b/Resource.js
@@ -411,7 +411,7 @@ class Resource {
     options = options || this.options;
 
     // Get the filters and omit the limit, skip, select, sort and populate.
-    const {limit, skip, select, sort, populate, ...filters} = req.query;
+    const {limit, skip, select, sort, populate, validationForm, ...filters} = req.query;
 
     // Iterate through each filter.
     Object.entries(filters).forEach(([name, values]) => {

From 27126ef11a9dedb88862fa99ce3f7c83e4736f53 Mon Sep 17 00:00:00 2001
From: Michal <xamien@seznam.cz>
Date: Tue, 18 Aug 2020 23:14:08 +0200
Subject: [PATCH 06/14] filter specific attributes in GET method

using query param select
---
 Resource.js | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/Resource.js b/Resource.js
index 25a0667..a3195e9 100644
--- a/Resource.js
+++ b/Resource.js
@@ -709,6 +709,13 @@ class Resource {
         debug.get(`Populate: ${populate}`);
         query.populate(populate);
       }
+      
+      // Filter specific attributes
+      const select = Resource.getParamQuery(req, 'select');
+      if (select) {
+        debug.get(`Select: ${select}`);
+        query.select(select);
+      }
 
       options.hooks.get.before.call(
         this,
@@ -726,22 +733,6 @@ class Resource {
               res,
               item,
               () => {
-                // Allow them to only return specified fields.
-                const select = Resource.getParamQuery(req, 'select');
-                if (select) {
-                  const newItem = {};
-                  // Always include the _id.
-                  if (item._id) {
-                    newItem._id = item._id;
-                  }
-                  select.split(' ').map(key => {
-                    key = key.trim();
-                    if (item.hasOwnProperty(key)) {
-                      newItem[key] = item[key];
-                    }
-                  });
-                  item = newItem;
-                }
                 Resource.setResponse(res, { status: 200, item: item }, next)
               }
             );

From f1dad47550ab2d936b5c7cb7ebfc37658f6d8779 Mon Sep 17 00:00:00 2001
From: Michal <xamien@seznam.cz>
Date: Tue, 25 Aug 2020 11:19:56 +0200
Subject: [PATCH 07/14] Update Resource.js

select query param fix for GET by _id and referenced attributes
---
 Resource.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Resource.js b/Resource.js
index a3195e9..aa1b203 100644
--- a/Resource.js
+++ b/Resource.js
@@ -714,7 +714,14 @@ class Resource {
       const select = Resource.getParamQuery(req, 'select');
       if (select) {
         debug.get(`Select: ${select}`);
-        query.select(select);
+        const selectSet = new Set(select.split(" "));
+        selectSet.forEach((element, index) => {
+          const dataArray = element.split(".data.");
+          if ( dataArray.length > 1 ) {
+            selectSet.add(`${dataArray[0]}._id`);
+          }
+        });
+        query.select(Array.from(selectSet).join(" "));
       }
 
       options.hooks.get.before.call(

From 12e7e45f90fd0e8b35bef695fa05e100f1470b18 Mon Sep 17 00:00:00 2001
From: siagimir <miroslav.siagi@gmail.com>
Date: Tue, 8 Sep 2020 14:13:46 +0200
Subject: [PATCH 08/14] Fixed sort function defined by query or pipelines.

---
 Resource.js | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/Resource.js b/Resource.js
index aa1b203..b63166d 100644
--- a/Resource.js
+++ b/Resource.js
@@ -631,12 +631,23 @@ class Resource {
           reqQuery.skip = pageRange.skip;
         }
 
-        // Add limit and skip to aggregation pipeline if present
-        if ( query.pipeline && query.pipeline.length > 0 ) {
+        // Add limit and skip to aggregation pipeline if present except sorting presence.
+        if ( query.pipeline && query.pipeline.length > 0 && !query.pipeline.find((object) => object['$sort'])) {
           query.pipeline.unshift({ $limit: reqQuery.limit });
           query.pipeline.unshift({ $skip: reqQuery.skip });
           reqQuery.skip = 0; // reset skip
         }
+        
+        if ( query.pipeline && query.pipeline.length > 0 && query.sort && Resource.getParamQuery(req, 'sort') ) {
+          const sort = Resource.getParamQuery(req, 'sort');
+          const negate = sort.indexOf('-') === 0;
+          const sortParam = negate ? sort.substr(1) : sort;
+          const sortObj = {
+            [sortParam]: negate ? -1 : 1
+          };
+          query.pipeline.unshift({ $sort: sortObj });
+          delete query.sort;
+        }
 
         // Next get the items within the index.
         const queryExec = query

From f6c2912ce5bdc3bb97285b9f4fc664654f2ef2e3 Mon Sep 17 00:00:00 2001
From: siagimir <miroslav.siagi@gmail.com>
Date: Thu, 1 Oct 2020 12:35:25 +0200
Subject: [PATCH 09/14] Added limit for performing an exact count query.

---
 Resource.js | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/Resource.js b/Resource.js
index b63166d..dc51490 100644
--- a/Resource.js
+++ b/Resource.js
@@ -524,9 +524,8 @@ class Resource {
     if (!utils.isEmpty(query._mongooseOptions) || !pipeline) {
       return query;
     }
-    const stages = [
+    const stagesMaxCount = [
       { $match: query.getQuery() },
-      ...pipeline,
       {
         $group: {
           _id : null,
@@ -534,13 +533,42 @@ class Resource {
         },
       },
     ];
+
     return {
       countDocuments(cb) {
-        query.model.aggregate(stages).exec((err, items) => {
+        query.model.aggregate(stagesMaxCount).exec((err, items) => {
           if (err) {
             return cb(err);
           }
-          return cb(null, items.length ? items[0].count : 0);
+
+          const maxCountLimit = process.env.MAX_COUNT_LIMIT;
+          const MAX_COUNT_LIMIT = maxCountLimit !== undefined ? maxCountLimit : 5000;
+
+          const itemsCount = items.length ? items[0].count : 0;
+          if (itemsCount > MAX_COUNT_LIMIT) {
+            return cb(null, MAX_COUNT_LIMIT);
+          }
+          else {
+
+            const stagesRealCount = [
+              {$match: query.getQuery()},
+              ...pipeline,
+              {
+                $group: {
+                  _id: null,
+                  count: {$sum: 1},
+                },
+              },
+            ];
+
+            query.model.aggregate(stagesRealCount).exec((errReal, itemsReal) => {
+              if (errReal) {
+                return cb(errReal);
+              }
+              return cb(null, itemsReal.length ? itemsReal[0].count : 0);
+            });
+
+          }
         });
       },
     };

From d3df138f26c936be7104c5e65c0df8055b5001c7 Mon Sep 17 00:00:00 2001
From: siagimir <miroslav.siagi@gmail.com>
Date: Thu, 1 Oct 2020 12:38:11 +0200
Subject: [PATCH 10/14] Added MAX_COUNT_LIMIT property description to a new
 section.

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index b8f9e06..0eef28f 100644
--- a/README.md
+++ b/README.md
@@ -83,6 +83,10 @@ Resource(app, route, name, model)
  - ***name*** - The name of the resource, which will then be used for the URL path of that resource.
  - ***model*** - The Mongoose Model for this interface.
 
+Properties
+----------------
+There is environment variable ```MAX_COUNT_LIMIT``` used when executing a count query. It represents a threshold for deciding whether to return a real number of records or the given limit. 
+
 Only exposing certain methods
 -------------------
 You can also expose only a certain amount of methods, by instead of using

From 29ac32a08db8c131ba3af664afa042cd5f80db04 Mon Sep 17 00:00:00 2001
From: Michal <xamien@seznam.cz>
Date: Wed, 21 Oct 2020 00:28:30 +0200
Subject: [PATCH 11/14] minimal limit for aggregations with nested query or
 sort

---
 Resource.js | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/Resource.js b/Resource.js
index dc51490..988eea8 100644
--- a/Resource.js
+++ b/Resource.js
@@ -660,10 +660,20 @@ class Resource {
         }
 
         // Add limit and skip to aggregation pipeline if present except sorting presence.
-        if ( query.pipeline && query.pipeline.length > 0 && !query.pipeline.find((object) => object['$sort'])) {
-          query.pipeline.unshift({ $limit: reqQuery.limit });
-          query.pipeline.unshift({ $skip: reqQuery.skip });
-          reqQuery.skip = 0; // reset skip
+        if ( query.pipeline && query.pipeline.length > 0 ) {
+
+          // minimal limit for aggregations with nested query or sort
+          const aggregateMinLimit = process.env.FORMIO_AGGREGATE_MIN_LIMIT;
+          const FORMIO_AGGREGATE_MIN_LIMIT = aggregateMinLimit !== undefined ? aggregateMinLimit : 1000;
+
+          const nestedMatchCount = query.pipeline.filter(item => (item.$sort || (item.$match && item.$match.$and && item.$match.$and.length > 1))).length;
+          query.pipeline.unshift({ $limit: (nestedMatchCount > 0 ? FORMIO_AGGREGATE_MIN_LIMIT : reqQuery.limit) });
+
+          // no nested filter => skip in pipeline
+          if ( nestedMatchCount <= 0 ) {
+            query.pipeline.unshift({ $skip: reqQuery.skip });
+            reqQuery.skip = 0; // reset skip
+          }
         }
         
         if ( query.pipeline && query.pipeline.length > 0 && query.sort && Resource.getParamQuery(req, 'sort') ) {

From db15c2397cdaa09373c8bc60baa4be624bacc0ab Mon Sep 17 00:00:00 2001
From: siagim <miroslav.siagi@notix.cz>
Date: Mon, 14 Dec 2020 18:01:54 +0100
Subject: [PATCH 12/14] Fixed parsing FORMIO_AGGREGATE_MIN_LIMIT env property's
 value.

---
 Resource.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Resource.js b/Resource.js
index 988eea8..f34ba36 100644
--- a/Resource.js
+++ b/Resource.js
@@ -663,8 +663,7 @@ class Resource {
         if ( query.pipeline && query.pipeline.length > 0 ) {
 
           // minimal limit for aggregations with nested query or sort
-          const aggregateMinLimit = process.env.FORMIO_AGGREGATE_MIN_LIMIT;
-          const FORMIO_AGGREGATE_MIN_LIMIT = aggregateMinLimit !== undefined ? aggregateMinLimit : 1000;
+          const FORMIO_AGGREGATE_MIN_LIMIT = parseInt(process.env.FORMIO_AGGREGATE_MIN_LIMIT) || 1000;
 
           const nestedMatchCount = query.pipeline.filter(item => (item.$sort || (item.$match && item.$match.$and && item.$match.$and.length > 1))).length;
           query.pipeline.unshift({ $limit: (nestedMatchCount > 0 ? FORMIO_AGGREGATE_MIN_LIMIT : reqQuery.limit) });
@@ -675,7 +674,7 @@ class Resource {
             reqQuery.skip = 0; // reset skip
           }
         }
-        
+
         if ( query.pipeline && query.pipeline.length > 0 && query.sort && Resource.getParamQuery(req, 'sort') ) {
           const sort = Resource.getParamQuery(req, 'sort');
           const negate = sort.indexOf('-') === 0;
@@ -758,7 +757,7 @@ class Resource {
         debug.get(`Populate: ${populate}`);
         query.populate(populate);
       }
-      
+
       // Filter specific attributes
       const select = Resource.getParamQuery(req, 'select');
       if (select) {

From f5dcb732fee600e143e82d057b8dddf5c560d1b9 Mon Sep 17 00:00:00 2001
From: Michal Kotek <michal.kotek@notix.cz>
Date: Wed, 6 Jan 2021 11:15:09 +0100
Subject: [PATCH 13/14] Fix skip parameter when populate exists.

---
 Resource.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Resource.js b/Resource.js
index f34ba36..bbdbda5 100644
--- a/Resource.js
+++ b/Resource.js
@@ -659,6 +659,8 @@ class Resource {
           reqQuery.skip = pageRange.skip;
         }
 
+        const populate = Resource.getParamQuery(req, 'populate');
+
         // Add limit and skip to aggregation pipeline if present except sorting presence.
         if ( query.pipeline && query.pipeline.length > 0 ) {
 
@@ -669,7 +671,7 @@ class Resource {
           query.pipeline.unshift({ $limit: (nestedMatchCount > 0 ? FORMIO_AGGREGATE_MIN_LIMIT : reqQuery.limit) });
 
           // no nested filter => skip in pipeline
-          if ( nestedMatchCount <= 0 ) {
+          if ( nestedMatchCount <= 0 && populate === "" ) {
             query.pipeline.unshift({ $skip: reqQuery.skip });
             reqQuery.skip = 0; // reset skip
           }
@@ -695,7 +697,6 @@ class Resource {
           .sort(Resource.getParamQuery(req, 'sort'));
 
         // Only call populate if they provide a populate query.
-        const populate = Resource.getParamQuery(req, 'populate');
         if (populate) {
           debug.index(`Populate: ${populate}`);
           queryExec.populate(populate);

From fc026572189b4715effdcab618159332da6c7a64 Mon Sep 17 00:00:00 2001
From: Michal <xamien@seznam.cz>
Date: Mon, 15 Feb 2021 21:40:28 +0100
Subject: [PATCH 14/14] fixing count optimization when populate

---
 Resource.js | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Resource.js b/Resource.js
index bbdbda5..e002d36 100644
--- a/Resource.js
+++ b/Resource.js
@@ -519,7 +519,7 @@ class Resource {
     return finalQuery;
   }
 
-  countQuery(query, pipeline) {
+  countQuery(query, pipeline, populate) {
     // We cannot use aggregation if mongoose special options are used... like populate.
     if (!utils.isEmpty(query._mongooseOptions) || !pipeline) {
       return query;
@@ -545,9 +545,12 @@ class Resource {
           const MAX_COUNT_LIMIT = maxCountLimit !== undefined ? maxCountLimit : 5000;
 
           const itemsCount = items.length ? items[0].count : 0;
-          if (itemsCount > MAX_COUNT_LIMIT) {
+          if (itemsCount > MAX_COUNT_LIMIT && populate === "") {
             return cb(null, MAX_COUNT_LIMIT);
           }
+          else if ( populate !== "" ) {
+            return cb(null, itemsCount);
+          }
           else {
 
             const stagesRealCount = [
@@ -625,8 +628,10 @@ class Resource {
       const countQuery = req.countQuery || req.modelQuery || req.model || this.model;
       const query = req.modelQuery || req.model || this.model;
 
+      const populate = Resource.getParamQuery(req, 'populate');
+      
       // First get the total count.
-      this.countQuery(countQuery.find(findQuery), countQuery.pipeline).countDocuments((err, count) => {
+      this.countQuery(countQuery.find(findQuery), countQuery.pipeline, populate).countDocuments((err, count) => {
         if (err) {
           debug.index(err);
           return Resource.setResponse(res, { status: 400, error: err }, next);
@@ -659,8 +664,6 @@ class Resource {
           reqQuery.skip = pageRange.skip;
         }
 
-        const populate = Resource.getParamQuery(req, 'populate');
-
         // Add limit and skip to aggregation pipeline if present except sorting presence.
         if ( query.pipeline && query.pipeline.length > 0 ) {