BFD-3517_duplicate POST Locust test suites

BFD-3517_duplicate POST Locust test suites

apps/utils/locust_tests/common/bfd_user_base.py

@@ -195,10 +195,50 @@ def get_by_url(self, url: str, headers: Optional[Mapping[str, str]] = None, name
                         self.url_pools[name] = []
                     self.url_pools[name].append(next_url)
 
+    def post_by_url(
+        self,
+        url: str,
+        headers: Optional[Mapping[str, str]] = None,
+        body: Optional[Mapping[str, str]] = None,
+        name: str = "",
+    ):
+        """Send one POST request and parse the response for pagination.
+
+        This method extends Locust's HttpUser::client.post() method to make creating the requests
+        nicer. Specifically, the query string parameters are specified as a separate dictionary
+        opposed to part of the path, the cert and verify arguments (which will never change) are
+        already set, and Cache-Control headers are automatically set to ensure caching is disabled.
+        """
+
+        safe_headers = {} if headers is None else headers
+        safe_body = {} if body is None else body
+
+        with self.client.post(
+            url,
+            headers={**safe_headers, "Cache-Control": "no-store, no-cache"},
+            data=safe_body,
+            name=name,  # type: ignore -- known Locust argument
+            catch_response=True,  # type: ignore -- known Locust argument
+        ) as response:
+            if response.status_code != 200:
+                if isinstance(response, ResponseContextManager):
+                    # pylint: disable=E1121
+                    response.failure(f"Status Code: {response.status_code}")
+                else:
+                    response.failure()
+            elif response.text:
+                # Check for valid "next" URLs that we can add to a URL pool.
+                next_url = BFDUserBase.__get_next_url(response.text)
+                if next_url is not None:
+                    if name not in self.url_pools:
+                        self.url_pools[name] = []
+                    self.url_pools[name].append(next_url)
+
     def run_task(
         self,
         url_callback: Callable,
         headers: Optional[Mapping[str, str]] = None,
+        body: Optional[Mapping[str, str]] = None,
         name: str = "",
     ):
         """Figure out which URL we should query next and query the server.
@@ -222,7 +262,10 @@ def run_task(
 
         if url is not None:
             # Run the test using the URL we found
-            self.get_by_url(url=url, headers=headers, name=name)
+            if body is None:
+                self.get_by_url(url=url, headers=headers, name=name)
+            else:
+                self.post_by_url(url=url, headers=headers, body=body, name=name)
         else:
             # If no URL is found, then this test isn't counted in statistics
 
@@ -251,6 +294,7 @@ def run_task_by_parameters(
         base_path: str,
         params: Optional[Mapping[str, Union[str, int, List[Any]]]] = None,
         headers: Optional[Mapping[str, str]] = None,
+        body: Optional[Mapping[str, str]] = None,
         name: str = "",
     ):
         """Run a task using a base path and parameters"""
@@ -261,7 +305,7 @@ def run_task_by_parameters(
         def make_url():
             return create_url_path(base_path, safe_params)
 
-        self.run_task(name=name, url_callback=make_url, headers=safe_headers)
+        self.run_task(name=name, url_callback=make_url, headers=safe_headers, body=body)
 
     # Helper Functions
 

apps/utils/locust_tests/common/db.py

@@ -81,6 +81,20 @@ def get_regression_hashed_mbis(uri: str) -> List[str]:
     return [str(r[0]) for r in _execute(uri, mbi_query)]
 
 
+def get_regression_mbis(uri: str) -> List[str]:
+    """Retrieves a list of hashed MBIs within the range of 20,000 contiguous synthetic
+    beneficiaries that exist in each environment. Returned list is sorted in ascending order
+
+    Args:
+        uri (str): Database URI
+
+    Returns:
+        List[str]: A list of synthetic hashed MBIs used for the regression suites
+    """
+    mbi_query = _get_regression_query('SELECT "mbi_num" FROM ccw.beneficiaries')
+    return [str(r[0]) for r in _execute(uri, mbi_query)]
+
+
 def get_regression_contract_ids(uri: str) -> List[Dict[str, str]]:
     """Retrieves a list of contract IDs within the range of 20,000 contiguous synthetic
     beneficiaries that exist in each environment. Returned list is sorted in ascending order, and
@@ -123,6 +137,21 @@ def get_regression_pac_hashed_mbis(uri: str) -> List[str]:
     return [str(r[0]) for r in _execute(uri, claims_mbis_query)]
 
 
+def get_regression_pac_mbis(uri: str) -> List[str]:
+    """Returns a list of MBI hashes within the set of static, synthetic PAC data
+
+    Args:
+        uri (str): The database connection string
+
+    Returns:
+        list[str]: A list of MBI hashes
+    """
+    claims_mbis_query = (
+        r"select mbi from rda.mbi_cache where regexp_like(mbi, '\dS.+') order by mbi limit 300"
+    )
+    return [str(r[0]) for r in _execute(uri, claims_mbis_query)]
+
+
 def get_bene_ids(uri: str, table_sample_pct: Optional[float] = None) -> List:
     """
     Return a list of bene IDs from the adjudicated beneficiary table
@@ -163,6 +192,31 @@ def get_hashed_mbis(uri: str, table_sample_pct: Optional[float] = None) -> List:
     return [str(r[0]) for r in _execute(uri, mbi_query)]
 
 
+def get_mbis(uri: str, table_sample_pct: Optional[float] = None) -> List:
+    """
+    Return a list of unique hashed MBIs from the adjudicated beneficiary table
+    """
+
+    if table_sample_pct is None:
+        table_sample_text = ""
+    else:
+        table_sample_text = f"TABLESAMPLE SYSTEM ({table_sample_pct}) "
+
+    # Handle possible hash collisions from duplicate hashes in beneficiaries_history
+    # by only taking MBI hashes that are distinct in both beneficiaries _and_
+    # benficiaries_history
+    mbi_query = (
+        f"SELECT beneficiaries.mbi_num FROM ccw.beneficiaries {table_sample_text} "
+        "    INNER JOIN ccw.beneficiaries_history "
+        "        ON beneficiaries.mbi_num = beneficiaries_history.mbi_num "
+        "    GROUP BY beneficiaries.mbi_num "
+        "    HAVING count(beneficiaries_history.mbi_num) = 1 "
+        f"LIMIT {LIMIT}"
+    )
+
+    return [str(r[0]) for r in _execute(uri, mbi_query)]
+
+
 def get_contract_ids(uri: str, table_sample_pct: Optional[float] = None) -> List:
     """
     Return a list of contract id / reference year pairs from the beneficiary
@@ -176,7 +230,7 @@ def get_contract_ids(uri: str, table_sample_pct: Optional[float] = None) -> List
 
     contract_id_query = (
         'SELECT "partd_contract_number_id", "year_month" '
-        'FROM ccw.beneficiary_monthly '
+        "FROM ccw.beneficiary_monthly "
         f"{table_sample_text}"
         f"LIMIT {LIMIT}"
     )
@@ -300,6 +354,113 @@ def get_pac_hashed_mbis(uri: str) -> List:
     return [str(r[0]) for r in reversed(_execute(uri, mbi_query))]
 
 
+def get_pac_mbis(uri: str) -> List:
+    """
+    Return a list of unique hashed MBIs that represent a diverse set of FISS and MCS
+    claims over a range of claim statuses.
+
+    We anticipate that fields will have a mixture of blank vs non-blank values based on the status codes received.
+
+    By selecting MBIs that are related to claims with varying status codes, we can get a good mixture of claim data
+    elements, better testing our FHIR transformers' ability to correctly render them.
+    """
+    per_status_max = int(LIMIT / 40)  # Based on ~40 distinct status values between FISS/MCS
+
+    sub_select_day_age = 30
+
+    """
+    selects FISS records only from the last N days
+    """
+    fiss_sub_query = (
+        "select * from rda.fiss_claims where last_updated > current_date - interval"
+        f" '{sub_select_day_age}' day and mbi_id is not null "
+    )
+
+    """
+    Selects FISS rows partitioned by status
+    """
+    fiss_partition_sub_query = (
+        "select fiss.*, ROW_NUMBER() "
+        "   over (partition by fiss.curr_status order by fiss.last_updated desc) "
+        f"from ({fiss_sub_query}) as fiss "
+    )
+
+    """
+    Selects the mbi ids from N of each FISS status
+    """
+    fiss_mbi_sub_query = (
+        "select fiss_partition.mbi_id "
+        f"from ({fiss_partition_sub_query}) fiss_partition "
+        f"where fiss_partition.row_number <= {per_status_max} "
+    )
+
+    """
+    selects MCS records only from the last N days
+    """
+    mcs_sub_query = (
+        "select * from rda.mcs_claims where last_updated > current_date - interval"
+        f" '{sub_select_day_age}' day and mbi_id is not null "
+    )
+
+    """
+    Selects MCS rows partitioned by status
+    """
+    mcs_partition_sub_query = (
+        "select mcs.*, ROW_NUMBER() "
+        "   over (partition by mcs.idr_status_code order by mcs.last_updated desc) "
+        f"from ({mcs_sub_query}) as mcs "
+    )
+
+    """
+    Selects the mbi ids from N of each MCS status
+    """
+    mcs_mbi_sub_query = (
+        "select mcs_partition.mbi_id "
+        f"from ({mcs_partition_sub_query}) mcs_partition "
+        f"where mcs_partition.row_number <= {per_status_max} "
+    )
+
+    """
+    Selects the distinct mbis from both fiss and mcs subqueries
+    """
+    distinct_type_status_mbis = (
+        "select distinct type_status.mbi_id "
+        f"from ({fiss_mbi_sub_query} union {mcs_mbi_sub_query}) as type_status "
+    )
+
+    mbi_query = (
+        # Get up to N distinct MBI hashes
+        "select mbi.mbi "
+        "from ( "
+        # Subquery sorts by source to weight 'filler' MBIs last
+        "	select union_select.mbi_id "
+        "	from ( "
+        # Select up to N of the newest claims for each distinct FISS and MCS status value
+        "		select src.mbi_id, 1 as source_order "
+        f"		from ({distinct_type_status_mbis}) src "
+        "		union "
+        # Select whatever MBIs as filler
+        "		select distinct mbi.mbi_id, 2 as source_order "
+        "		from ( "
+        "           select recent_mbis.* "
+        "           from rda.mbi_cache as recent_mbis "
+        f"		    where recent_mbis.mbi_id not in ({distinct_type_status_mbis}) "
+        "           order by recent_mbis.last_updated desc "
+        "       ) as mbi "
+        f"		limit {LIMIT} "
+        "	) as union_select "
+        "	order by union_select.source_order "
+        ") sources "
+        "left join rda.mbi_cache as mbi on mbi.mbi_id = sources.mbi_id "
+        f"limit {LIMIT}"
+    )
+
+    # intentionally reversing the query results, as the important mbis to test
+    # will be at the beginning of the result set and BFDUserBase will pop items
+    # off of the end of the list
+    return [str(r[0]) for r in reversed(_execute(uri, mbi_query))]
+
+
 def get_pac_hashed_mbis_smoketest(uri: str) -> list[str]:
     """Gets the top LIMIT MBI hashes from the rda table's MBI cache for use with the PACA smoketests
 
@@ -312,3 +473,17 @@ def get_pac_hashed_mbis_smoketest(uri: str) -> list[str]:
     smoketest_mbi_query = f"select hash from rda.mbi_cache limit {LIMIT}"
 
     return [str(r[0]) for r in _execute(uri, smoketest_mbi_query)]
+
+
+def get_pac_mbis_smoketest(uri: str) -> list[str]:
+    """Gets the top LIMIT MBI hashes from the rda table's MBI cache for use with the PACA smoketests
+
+    Args:
+        uri (str): The database connection string
+
+    Returns:
+        list[str]: A list of MBI hashes
+    """
+    smoketest_mbi_query = f"select mbi from rda.mbi_cache limit {LIMIT}"
+
+    return [str(r[0]) for r in _execute(uri, smoketest_mbi_query)]