From bf3bf2e12cbdffe4f474227366725fa496b55abe Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:31:41 +0200 Subject: [PATCH 01/49] add metric and variant classes --- cluster_experiments/metric.py | 55 ++++++++++++++++++++++++++++++++++ cluster_experiments/variant.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 cluster_experiments/metric.py create mode 100644 cluster_experiments/variant.py diff --git a/cluster_experiments/metric.py b/cluster_experiments/metric.py new file mode 100644 index 0000000..3803ad2 --- /dev/null +++ b/cluster_experiments/metric.py @@ -0,0 +1,55 @@ +class Metric: + """ + A class used to represent a Metric with an alias and components. + + Attributes + ---------- + alias : str + A string representing the alias of the metric + components : tuple + A tuple of strings representing the components of the metric + + Methods + ------- + __init__(self, alias: str, components: tuple): + Initializes the Metric with the provided alias and components. + _validate_inputs(alias: str, components: tuple): + Validates the inputs for the Metric class. + """ + + def __init__(self, alias: str, components: tuple): + """ + Parameters + ---------- + alias : str + The alias of the metric + components : tuple + A tuple of strings representing the components of the metric + """ + self._validate_inputs(alias, components) + self.alias = alias + self.components = components + + @staticmethod + def _validate_inputs(alias: str, components: tuple): + """ + Validates the inputs for the Metric class. + + Parameters + ---------- + alias : str + The alias of the metric + components : tuple + A tuple of strings representing the components of the metric + + Raises + ------ + TypeError + If the alias is not a string or if components is not a tuple of strings. + """ + if not isinstance(alias, str): + raise TypeError("Alias must be a string") + if not isinstance(components, tuple) or not all( + isinstance(comp, str) for comp in components + ): + raise TypeError("Components must be a tuple of strings") diff --git a/cluster_experiments/variant.py b/cluster_experiments/variant.py new file mode 100644 index 0000000..1bc84fc --- /dev/null +++ b/cluster_experiments/variant.py @@ -0,0 +1,53 @@ +class Variant: + """ + A class used to represent a Variant with a name and a control flag. + + Attributes + ---------- + name : str + The name of the variant + is_control : bool + A boolean indicating if the variant is a control variant + + Methods + ------- + __init__(self, name: str, is_control: bool): + Initializes the Variant with the provided name and control flag. + _validate_inputs(name: str, is_control: bool): + Validates the inputs for the Variant class. + """ + + def __init__(self, name: str, is_control: bool): + """ + Parameters + ---------- + name : str + The name of the variant + is_control : bool + A boolean indicating if the variant is a control variant + """ + self._validate_inputs(name, is_control) + self.name = name + self.is_control = is_control + + @staticmethod + def _validate_inputs(name: str, is_control: bool): + """ + Validates the inputs for the Variant class. + + Parameters + ---------- + name : str + The name of the variant + is_control : bool + A boolean indicating if the variant is a control variant + + Raises + ------ + TypeError + If the name is not a string or if is_control is not a boolean. + """ + if not isinstance(name, str): + raise TypeError("Name must be a string") + if not isinstance(is_control, bool): + raise TypeError("is_control must be a boolean") From 5514c7e55ab92fe6edff383f20832b2c0ebe132d Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:56:15 +0200 Subject: [PATCH 02/49] add dimension class --- cluster_experiments/dimension.py | 58 ++++++++++++++++++++++++++++++++ cluster_experiments/metric.py | 4 +-- cluster_experiments/variant.py | 4 +-- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 cluster_experiments/dimension.py diff --git a/cluster_experiments/dimension.py b/cluster_experiments/dimension.py new file mode 100644 index 0000000..1306cbc --- /dev/null +++ b/cluster_experiments/dimension.py @@ -0,0 +1,58 @@ +from typing import List + + +class Dimension: + """ + A class used to represent a Dimension with a name and values. + + Attributes + ---------- + name : str + The name of the dimension + values : List[str] + A list of strings representing the possible values of the dimension + + Methods + ------- + __init__(self, name: str, values: List[str]): + Initializes the Dimension with the provided name and values. + _validate_inputs(name: str, values: List[str]): + Validates the inputs for the Dimension class. + """ + + def __init__(self, name: str, values: List[str]): + """ + Parameters + ---------- + name : str + The name of the dimension + values : List[str] + A list of strings representing the possible values of the dimension + """ + self._validate_inputs(name, values) + self.name = name + self.values = values + + @staticmethod + def _validate_inputs(name: str, values: List[str]): + """ + Validates the inputs for the Dimension class. + + Parameters + ---------- + name : str + The name of the dimension + values : List[str] + A list of strings representing the possible values of the dimension + + Raises + ------ + TypeError + If the name is not a string or if values is not a list of strings. + """ + if not isinstance(name, str): + raise TypeError("Dimension name must be a string") + if not isinstance(values, list) or not all( + isinstance(val, str) for val in values + ): + raise TypeError("Dimension values must be a list of strings") diff --git a/cluster_experiments/metric.py b/cluster_experiments/metric.py index 3803ad2..6917693 100644 --- a/cluster_experiments/metric.py +++ b/cluster_experiments/metric.py @@ -48,8 +48,8 @@ def _validate_inputs(alias: str, components: tuple): If the alias is not a string or if components is not a tuple of strings. """ if not isinstance(alias, str): - raise TypeError("Alias must be a string") + raise TypeError("Metric alias must be a string") if not isinstance(components, tuple) or not all( isinstance(comp, str) for comp in components ): - raise TypeError("Components must be a tuple of strings") + raise TypeError("Metric components must be a tuple of strings") diff --git a/cluster_experiments/variant.py b/cluster_experiments/variant.py index 1bc84fc..ad8f3c1 100644 --- a/cluster_experiments/variant.py +++ b/cluster_experiments/variant.py @@ -48,6 +48,6 @@ def _validate_inputs(name: str, is_control: bool): If the name is not a string or if is_control is not a boolean. """ if not isinstance(name, str): - raise TypeError("Name must be a string") + raise TypeError("Variant name must be a string") if not isinstance(is_control, bool): - raise TypeError("is_control must be a boolean") + raise TypeError("Variant is_control must be a boolean") From 0a786b3a4e7f49464d8208af9f57cecc870d48e9 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:08:12 +0200 Subject: [PATCH 03/49] add hypothesis test class --- cluster_experiments/hypothesis_test.py | 97 ++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 cluster_experiments/hypothesis_test.py diff --git a/cluster_experiments/hypothesis_test.py b/cluster_experiments/hypothesis_test.py new file mode 100644 index 0000000..cc621d7 --- /dev/null +++ b/cluster_experiments/hypothesis_test.py @@ -0,0 +1,97 @@ +from typing import List, Optional + +from cluster_experiments.dimension import ( + Dimension, +) +from cluster_experiments.experiment_analysis import ExperimentAnalysis +from cluster_experiments.metric import Metric + + +class HypothesisTest: + """ + A class used to represent a Hypothesis Test with a metric, analysis, optional analysis configuration, and optional dimensions. + + Attributes + ---------- + metric : Metric + An instance of the Metric class + analysis : ExperimentAnalysis + An instance of the ExperimentAnalysis class + analysis_config : Optional[dict] + An optional dictionary representing the configuration for the analysis + dimensions : Optional[List[Dimension]] + An optional list of Dimension instances + + Methods + ------- + __init__(self, metric: Metric, analysis: ExperimentAnalysis, analysis_config: Optional[dict] = None, dimensions: Optional[List[Dimension]] = None): + Initializes the HypothesisTest with the provided metric, analysis, and optional analysis configuration and dimensions. + _validate_inputs(metric: Metric, analysis: ExperimentAnalysis, analysis_config: Optional[dict], dimensions: Optional[List[Dimension]]): + Validates the inputs for the HypothesisTest class. + """ + + def __init__( + self, + metric: Metric, + analysis: ExperimentAnalysis, + analysis_config: Optional[dict] = None, + dimensions: Optional[List[Dimension]] = None, + ): + """ + Parameters + ---------- + metric : Metric + An instance of the Metric class + analysis : ExperimentAnalysis + An instance of the ExperimentAnalysis class + analysis_config : Optional[dict] + An optional dictionary representing the configuration for the analysis + dimensions : Optional[List[Dimension]] + An optional list of Dimension instances + """ + self._validate_inputs(metric, analysis, analysis_config, dimensions) + self.metric = metric + self.analysis = analysis + self.analysis_config = analysis_config or {} + self.dimensions = dimensions or [] + + @staticmethod + def _validate_inputs( + metric: Metric, + analysis: ExperimentAnalysis, + analysis_config: Optional[dict], + dimensions: Optional[List[Dimension]], + ): + """ + Validates the inputs for the HypothesisTest class. + + Parameters + ---------- + metric : Metric + An instance of the Metric class + analysis : ExperimentAnalysis + An instance of the ExperimentAnalysis class + analysis_config : Optional[dict] + An optional dictionary representing the configuration for the analysis + dimensions : Optional[List[Dimension]] + An optional list of Dimension instances + + Raises + ------ + TypeError + If metric is not an instance of Metric, if analysis is not an instance of ExperimentAnalysis, + if analysis_config is not a dictionary (when provided), or if dimensions is not a list of Dimension instances (when provided). + """ + if not isinstance(metric, Metric): + raise TypeError("Metric must be an instance of Metric") + if not isinstance(analysis, ExperimentAnalysis): + raise TypeError("Analysis must be an instance of ExperimentAnalysis") + if analysis_config is not None and not isinstance(analysis_config, dict): + raise TypeError("Analysis_config must be a dictionary if provided") + if dimensions is not None and ( + not isinstance(dimensions, list) + or not all(isinstance(dim, Dimension) for dim in dimensions) + ): + raise TypeError( + "Dimensions must be a list of Dimension instances if provided" + ) From d830e8e35837fb70e8580da5f94257a403a5c119 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:08:32 +0200 Subject: [PATCH 04/49] add hypothesis test class --- cluster_experiments/hypothesis_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cluster_experiments/hypothesis_test.py b/cluster_experiments/hypothesis_test.py index cc621d7..fe199f0 100644 --- a/cluster_experiments/hypothesis_test.py +++ b/cluster_experiments/hypothesis_test.py @@ -1,8 +1,6 @@ from typing import List, Optional -from cluster_experiments.dimension import ( - Dimension, -) +from cluster_experiments.dimension import Dimension from cluster_experiments.experiment_analysis import ExperimentAnalysis from cluster_experiments.metric import Metric From 1477e8a11fd5e1fe09fc4acba34a20067164bdf5 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:14:18 +0200 Subject: [PATCH 05/49] add analysis plan class --- cluster_experiments/analysis_plan.py | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 cluster_experiments/analysis_plan.py diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py new file mode 100644 index 0000000..dc3a8f4 --- /dev/null +++ b/cluster_experiments/analysis_plan.py @@ -0,0 +1,69 @@ +from typing import List + +from cluster_experiments.hypothesis_test import HypothesisTest +from cluster_experiments.variant import Variant + + +class AnalysisPlan: + """ + A class used to represent an Analysis Plan with a list of hypothesis tests and a list of variants. + + Attributes + ---------- + tests : List[HypothesisTest] + A list of HypothesisTest instances + variants : List[Variant] + A list of Variant instances + + Methods + ------- + __init__(self, tests: List[HypothesisTest], variants: List[Variant]): + Initializes the AnalysisPlan with the provided list of hypothesis tests and variants. + _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): + Validates the inputs for the AnalysisPlan class. + """ + + def __init__(self, tests: List[HypothesisTest], variants: List[Variant]): + """ + Parameters + ---------- + tests : List[HypothesisTest] + A list of HypothesisTest instances + variants : List[Variant] + A list of Variant instances + """ + self._validate_inputs(tests, variants) + self.tests = tests + self.variants = variants + + @staticmethod + def _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): + """ + Validates the inputs for the AnalysisPlan class. + + Parameters + ---------- + tests : List[HypothesisTest] + A list of HypothesisTest instances + variants : List[Variant] + A list of Variant instances + + Raises + ------ + TypeError + If tests is not a list of HypothesisTest instances or if variants is not a list of Variant instances. + ValueError + If tests or variants are empty lists. + """ + if not isinstance(tests, list) or not all( + isinstance(test, HypothesisTest) for test in tests + ): + raise TypeError("Tests must be a list of HypothesisTest instances") + if not isinstance(variants, list) or not all( + isinstance(variant, Variant) for variant in variants + ): + raise TypeError("Variants must be a list of Variant instances") + if not tests: + raise ValueError("Tests list cannot be empty") + if not variants: + raise ValueError("Variants list cannot be empty") From 0f51902e91f5622daebd24aba1a70366b0f1302f Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:47:11 +0200 Subject: [PATCH 06/49] add analysis results data structures --- cluster_experiments/analysis_results.py | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 cluster_experiments/analysis_results.py diff --git a/cluster_experiments/analysis_results.py b/cluster_experiments/analysis_results.py new file mode 100644 index 0000000..3cd87b3 --- /dev/null +++ b/cluster_experiments/analysis_results.py @@ -0,0 +1,96 @@ +from dataclasses import dataclass +from typing import List + +import pandas as pd + + +@dataclass +class HypothesisTestResults: + """ + A dataclass used to represent the results of a Hypothesis Test. + + Attributes + ---------- + metric_alias : str + The alias of the metric used in the test + control_variant_name : str + The name of the control variant + treatment_variant_name : str + The name of the treatment variant + control_variant_mean : float + The mean value of the control variant + treatment_variant_mean : float + The mean value of the treatment variant + analysis_type : str + The type of analysis performed + ate : float + The average treatment effect + ate_ci_lower : float + The lower bound of the confidence interval for the ATE + ate_ci_upper : float + The upper bound of the confidence interval for the ATE + p_value : float + The p-value of the test + std_error : float + The standard error of the test + dimension_name : str + The name of the dimension + dimension_value : str + The value of the dimension + """ + + metric_alias: str + control_variant_name: str + treatment_variant_name: str + control_variant_mean: float + treatment_variant_mean: float + analysis_type: str + ate: float + ate_ci_lower: float + ate_ci_upper: float + p_value: float + std_error: float + dimension_name: str + dimension_value: str + + +class AnalysisPlanResults(pd.DataFrame): + """ + A class used to represent the results of an Analysis Plan as a pandas DataFrame. + + Methods + ------- + add_results(results: List[HypothesisTestResults]): + Adds a list of new results to the DataFrame. + """ + + def __init__(self, *args, **kwargs): + columns = [ + "metric_alias", + "control_variant_name", + "treatment_variant_name", + "control_variant_mean", + "treatment_variant_mean", + "analysis_type", + "ate", + "ate_ci_lower", + "ate_ci_upper", + "p_value", + "std_error", + "dimension_name", + "dimension_value", + ] + super().__init__(*args, columns=columns, **kwargs) + + def add_results(self, results: List[HypothesisTestResults]): + """ + Adds a list of new results to the DataFrame. + + Parameters + ---------- + results : List[HypothesisTestResults] + The list of results to be added to the DataFrame + """ + new_data = [result.__dict__ for result in results] + new_df = pd.DataFrame(new_data) + self.append(new_df, ignore_index=True) From 4524381603282833d0008898e0ddc4bd1a1cde14 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:02:12 +0200 Subject: [PATCH 07/49] add analysis plan analyze signature --- cluster_experiments/analysis_plan.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index dc3a8f4..4ecfc65 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -1,5 +1,8 @@ from typing import List +import pandas as pd + +from cluster_experiments.analysis_results import AnalysisPlanResults from cluster_experiments.hypothesis_test import HypothesisTest from cluster_experiments.variant import Variant @@ -67,3 +70,12 @@ def _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): raise ValueError("Tests list cannot be empty") if not variants: raise ValueError("Variants list cannot be empty") + + def analyze( + self, exp_data: pd.DataFrame, pre_exp_data: pd.DataFrame, alpha=0.05 + ) -> AnalysisPlanResults: + ... + # add methods to prepare the filtered dataset based on variants and slicers + # add methods to run the analysis for each of the hypothesis tests, given a filtered dataset + # store each row as a HypothesisTestResults object + # wrap all results in an AnalysisPlanResults object From 45fd7a2e87d2e19e8fecd785124e4db80c9efa9d Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:15:31 +0200 Subject: [PATCH 08/49] add analysis plan analyze draft --- cluster_experiments/analysis_plan.py | 128 ++++++++++++++++++++++++- cluster_experiments/hypothesis_test.py | 26 ++--- 2 files changed, 137 insertions(+), 17 deletions(-) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index 4ecfc65..6b0d436 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -2,8 +2,12 @@ import pandas as pd -from cluster_experiments.analysis_results import AnalysisPlanResults +from cluster_experiments.analysis_results import ( + AnalysisPlanResults, + HypothesisTestResults, +) from cluster_experiments.hypothesis_test import HypothesisTest +from cluster_experiments.power_config import analysis_mapping from cluster_experiments.variant import Variant @@ -26,7 +30,12 @@ class AnalysisPlan: Validates the inputs for the AnalysisPlan class. """ - def __init__(self, tests: List[HypothesisTest], variants: List[Variant]): + def __init__( + self, + tests: List[HypothesisTest], + variants: List[Variant], + variant_col: str = "treatment", + ): """ Parameters ---------- @@ -34,13 +43,18 @@ def __init__(self, tests: List[HypothesisTest], variants: List[Variant]): A list of HypothesisTest instances variants : List[Variant] A list of Variant instances + variant_col : str + The name of the column containing the variant names. """ - self._validate_inputs(tests, variants) + self._validate_inputs(tests, variants, variant_col) self.tests = tests self.variants = variants + self.variant_col = variant_col @staticmethod - def _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): + def _validate_inputs( + tests: List[HypothesisTest], variants: List[Variant], variant_col: str + ): """ Validates the inputs for the AnalysisPlan class. @@ -50,6 +64,8 @@ def _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): A list of HypothesisTest instances variants : List[Variant] A list of Variant instances + variant_col : str + The name of the column containing the variant names. Raises ------ @@ -66,6 +82,8 @@ def _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): isinstance(variant, Variant) for variant in variants ): raise TypeError("Variants must be a list of Variant instances") + if not isinstance(variant_col, str): + raise TypeError("Variant_col must be a string") if not tests: raise ValueError("Tests list cannot be empty") if not variants: @@ -79,3 +97,105 @@ def analyze( # add methods to run the analysis for each of the hypothesis tests, given a filtered dataset # store each row as a HypothesisTestResults object # wrap all results in an AnalysisPlanResults object + + # ----- + + # add all kind of checks on the inputs at the beginning using the data structures + # todo: ... + # do it before running the computations below + + results = AnalysisPlanResults() + treatment_variants: List[Variant] = self.get_treatment_variants() + control_variant: Variant = self.get_control_variant() + + for test in self.tests: + # add cupac handler here + for treatment_variant in treatment_variants: + for dimension in test.dimensions: + for dimension_value in list(set(dimension.values)): + prepared_df = self.prepare_data( + data=exp_data, + variant_col=self.variant_col, + treatment_variant=treatment_variant, + control_variant=control_variant, + dimension_name=dimension.name, + dimension_value=dimension_value, + ) + + analysis_class = analysis_mapping[test.analysis_type] + experiment_analysis = analysis_class(**test.analysis_config) + experiment_analysis.target_col = test.metric.alias + experiment_analysis.treatment_col = (self.variant_col,) + experiment_analysis.treatment = treatment_variant + + HypothesisTestResults( + metric_alias=test.metric.alias, + control_variant_name=control_variant.name, + treatment_variant_name=treatment_variant.name, + control_variant_mean=0.5, # todo: add method + treatment_variant_mean=0.6, # todo: add method + analysis_type=test.analysis_type, + ate=experiment_analysis.get_point_estimate(df=prepared_df), + ate_ci_lower=0.1, # todo: add method + ate_ci_upper=0.2, # todo: add method + p_value=experiment_analysis.get_point_estimate( + df=prepared_df + ), + std_error=experiment_analysis.get_standard_error( + df=prepared_df + ), + dimension_name=dimension.name, + dimension_value=dimension_value, + ) + + return results + + def prepare_data( + self, + data: pd.DataFrame, + variant_col: str, + treatment_variant: Variant, + control_variant: Variant, + dimension_name: str, + dimension_value: str, + ) -> pd.DataFrame: + """ + Prepares the data for the experiment analysis pipeline + """ + prepared_df = data.copy() + + prepared_df = prepared_df.query( + f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'')" + ).query(f"{dimension_name} == '{dimension_value}'") + + return prepared_df + + def get_control_variant(self) -> Variant: + """ + Returns the control variant from the list of variants. Raises an error if no control variant is found. + + Returns + ------- + Variant + The control variant + + Raises + ------ + ValueError + If no control variant is found + """ + for variant in self.variants: + if variant.is_control: + return variant + raise ValueError("No control variant found") + + def get_treatment_variants(self) -> List[Variant]: + """ + Returns the treatment variants from the list of variants. + + Returns + ------- + List[Variant] + A list of treatment variants + """ + return [variant for variant in self.variants if not variant.is_control] diff --git a/cluster_experiments/hypothesis_test.py b/cluster_experiments/hypothesis_test.py index fe199f0..10f2b2e 100644 --- a/cluster_experiments/hypothesis_test.py +++ b/cluster_experiments/hypothesis_test.py @@ -1,7 +1,6 @@ from typing import List, Optional from cluster_experiments.dimension import Dimension -from cluster_experiments.experiment_analysis import ExperimentAnalysis from cluster_experiments.metric import Metric @@ -31,7 +30,7 @@ class HypothesisTest: def __init__( self, metric: Metric, - analysis: ExperimentAnalysis, + analysis_type: str, analysis_config: Optional[dict] = None, dimensions: Optional[List[Dimension]] = None, ): @@ -40,23 +39,23 @@ def __init__( ---------- metric : Metric An instance of the Metric class - analysis : ExperimentAnalysis - An instance of the ExperimentAnalysis class + analysis_type : str + string mapper to an ExperimentAnalysis analysis_config : Optional[dict] An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances """ - self._validate_inputs(metric, analysis, analysis_config, dimensions) + self._validate_inputs(metric, analysis_type, analysis_config, dimensions) self.metric = metric - self.analysis = analysis + self.analysis_type = analysis_type self.analysis_config = analysis_config or {} - self.dimensions = dimensions or [] + self.dimensions = dimensions or [Dimension(name="total", values=["total"])] @staticmethod def _validate_inputs( metric: Metric, - analysis: ExperimentAnalysis, + analysis_type: str, analysis_config: Optional[dict], dimensions: Optional[List[Dimension]], ): @@ -67,8 +66,8 @@ def _validate_inputs( ---------- metric : Metric An instance of the Metric class - analysis : ExperimentAnalysis - An instance of the ExperimentAnalysis class + analysis_type : str + string mapper to an ExperimentAnalysis analysis_config : Optional[dict] An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] @@ -77,13 +76,14 @@ def _validate_inputs( Raises ------ TypeError - If metric is not an instance of Metric, if analysis is not an instance of ExperimentAnalysis, + If metric is not an instance of Metric, if analysis_type is not an instance of string, if analysis_config is not a dictionary (when provided), or if dimensions is not a list of Dimension instances (when provided). """ if not isinstance(metric, Metric): raise TypeError("Metric must be an instance of Metric") - if not isinstance(analysis, ExperimentAnalysis): - raise TypeError("Analysis must be an instance of ExperimentAnalysis") + if not isinstance(analysis_type, str): + raise TypeError("Analysis must be a string") + # todo: add better check for analysis_type allowed values if analysis_config is not None and not isinstance(analysis_config, dict): raise TypeError("Analysis_config must be a dictionary if provided") if dimensions is not None and ( From 278929ca2d769fd540441d9678a64568295c7808 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:53:41 +0200 Subject: [PATCH 09/49] change signature for metric components --- cluster_experiments/metric.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cluster_experiments/metric.py b/cluster_experiments/metric.py index 6917693..8f10db4 100644 --- a/cluster_experiments/metric.py +++ b/cluster_experiments/metric.py @@ -1,3 +1,6 @@ +from typing import Union + + class Metric: """ A class used to represent a Metric with an alias and components. @@ -6,8 +9,8 @@ class Metric: ---------- alias : str A string representing the alias of the metric - components : tuple - A tuple of strings representing the components of the metric + components : str or tuple + A string or a tuple of strings representing the components of the metric Methods ------- @@ -17,14 +20,14 @@ class Metric: Validates the inputs for the Metric class. """ - def __init__(self, alias: str, components: tuple): + def __init__(self, alias: str, components: Union[tuple, str]): """ Parameters ---------- alias : str The alias of the metric - components : tuple - A tuple of strings representing the components of the metric + components : tuple or str + A string or a tuple of strings representing the components of the metric """ self._validate_inputs(alias, components) self.alias = alias @@ -39,7 +42,7 @@ def _validate_inputs(alias: str, components: tuple): ---------- alias : str The alias of the metric - components : tuple + components : tuple or str A tuple of strings representing the components of the metric Raises @@ -50,6 +53,7 @@ def _validate_inputs(alias: str, components: tuple): if not isinstance(alias, str): raise TypeError("Metric alias must be a string") if not isinstance(components, tuple) or not all( - isinstance(comp, str) for comp in components + isinstance(comp, str) + for comp in components or not isinstance(components, str) ): - raise TypeError("Metric components must be a tuple of strings") + raise TypeError("Metric components must be a string or a tuple of strings") From 7fb2a1584c447e97c7df7a7f80f1c8030befb20a Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sat, 20 Jul 2024 03:59:01 +0200 Subject: [PATCH 10/49] fix bugs --- cluster_experiments/analysis_plan.py | 65 ++++++++++++++----------- cluster_experiments/analysis_results.py | 5 +- cluster_experiments/metric.py | 8 +-- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index 6b0d436..643e554 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -1,9 +1,9 @@ from typing import List import pandas as pd +from pandas import DataFrame from cluster_experiments.analysis_results import ( - AnalysisPlanResults, HypothesisTestResults, ) from cluster_experiments.hypothesis_test import HypothesisTest @@ -90,8 +90,9 @@ def _validate_inputs( raise ValueError("Variants list cannot be empty") def analyze( - self, exp_data: pd.DataFrame, pre_exp_data: pd.DataFrame, alpha=0.05 - ) -> AnalysisPlanResults: + self, + exp_data: pd.DataFrame, # , pre_exp_data: Optional[pd.DataFrame], alpha=0.05 + ) -> DataFrame: ... # add methods to prepare the filtered dataset based on variants and slicers # add methods to run the analysis for each of the hypothesis tests, given a filtered dataset @@ -104,7 +105,7 @@ def analyze( # todo: ... # do it before running the computations below - results = AnalysisPlanResults() + test_results = [] treatment_variants: List[Variant] = self.get_treatment_variants() control_variant: Variant = self.get_control_variant() @@ -123,32 +124,38 @@ def analyze( ) analysis_class = analysis_mapping[test.analysis_type] - experiment_analysis = analysis_class(**test.analysis_config) - experiment_analysis.target_col = test.metric.alias - experiment_analysis.treatment_col = (self.variant_col,) - experiment_analysis.treatment = treatment_variant - - HypothesisTestResults( - metric_alias=test.metric.alias, - control_variant_name=control_variant.name, - treatment_variant_name=treatment_variant.name, - control_variant_mean=0.5, # todo: add method - treatment_variant_mean=0.6, # todo: add method - analysis_type=test.analysis_type, - ate=experiment_analysis.get_point_estimate(df=prepared_df), - ate_ci_lower=0.1, # todo: add method - ate_ci_upper=0.2, # todo: add method - p_value=experiment_analysis.get_point_estimate( - df=prepared_df - ), - std_error=experiment_analysis.get_standard_error( - df=prepared_df - ), - dimension_name=dimension.name, - dimension_value=dimension_value, + experiment_analysis = analysis_class( + **test.analysis_config, + target_col=test.metric.components, # todo: add support for ratio and delta method + treatment_col=self.variant_col, + treatment=treatment_variant.name, + ) + + ate = experiment_analysis.get_point_estimate(df=prepared_df) + p_value = experiment_analysis.get_pvalue(df=prepared_df) + std_error = experiment_analysis.get_standard_error( + df=prepared_df + ) + + test_results.append( + HypothesisTestResults( + metric_alias=test.metric.alias, + control_variant_name=control_variant.name, + treatment_variant_name=treatment_variant.name, + control_variant_mean=0.5, # todo: add method + treatment_variant_mean=0.6, # todo: add method + analysis_type=test.analysis_type, + ate=ate, + ate_ci_lower=0.1, # todo: add method + ate_ci_upper=0.2, # todo: add method + p_value=p_value, + std_error=std_error, + dimension_name=dimension.name, + dimension_value=dimension_value, + ) ) - return results + return pd.DataFrame([test_result.__dict__ for test_result in test_results]) def prepare_data( self, @@ -165,7 +172,7 @@ def prepare_data( prepared_df = data.copy() prepared_df = prepared_df.query( - f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'')" + f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])" ).query(f"{dimension_name} == '{dimension_value}'") return prepared_df diff --git a/cluster_experiments/analysis_results.py b/cluster_experiments/analysis_results.py index 3cd87b3..e90637a 100644 --- a/cluster_experiments/analysis_results.py +++ b/cluster_experiments/analysis_results.py @@ -82,7 +82,8 @@ def __init__(self, *args, **kwargs): ] super().__init__(*args, columns=columns, **kwargs) - def add_results(self, results: List[HypothesisTestResults]): + @staticmethod + def add_results(results: List[HypothesisTestResults]) -> pd.DataFrame: """ Adds a list of new results to the DataFrame. @@ -93,4 +94,4 @@ def add_results(self, results: List[HypothesisTestResults]): """ new_data = [result.__dict__ for result in results] new_df = pd.DataFrame(new_data) - self.append(new_df, ignore_index=True) + return new_df diff --git a/cluster_experiments/metric.py b/cluster_experiments/metric.py index 8f10db4..8539840 100644 --- a/cluster_experiments/metric.py +++ b/cluster_experiments/metric.py @@ -53,7 +53,9 @@ def _validate_inputs(alias: str, components: tuple): if not isinstance(alias, str): raise TypeError("Metric alias must be a string") if not isinstance(components, tuple) or not all( - isinstance(comp, str) - for comp in components or not isinstance(components, str) + isinstance(comp, str) for comp in components ): - raise TypeError("Metric components must be a string or a tuple of strings") + if not isinstance(components, str): + raise TypeError( + "Metric components must be a string or a tuple of strings" + ) From 5f9e1a203c114555ccb49c68b92fe287939311a1 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sun, 21 Jul 2024 16:15:46 +0200 Subject: [PATCH 11/49] improve metric system --- cluster_experiments/analysis_plan.py | 64 +++++-- cluster_experiments/analysis_results.py | 30 ++-- cluster_experiments/metric.py | 216 ++++++++++++++++++++---- 3 files changed, 258 insertions(+), 52 deletions(-) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index 643e554..663c418 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -4,6 +4,7 @@ from pandas import DataFrame from cluster_experiments.analysis_results import ( + AnalysisPlanResults, HypothesisTestResults, ) from cluster_experiments.hypothesis_test import HypothesisTest @@ -110,8 +111,21 @@ def analyze( control_variant: Variant = self.get_control_variant() for test in self.tests: - # add cupac handler here + # todo: add cupac handler here + analysis_class = analysis_mapping[test.analysis_type] + target_col = test.metric.get_target_column_from_metric() + for treatment_variant in treatment_variants: + + analysis_config_final = self.prepare_analysis_config( + initial_analysis_config=test.analysis_config, + target_col=target_col, + treatment_col=self.variant_col, + treatment=treatment_variant.name, + ) + + experiment_analysis = analysis_class(**analysis_config_final) + for dimension in test.dimensions: for dimension_value in list(set(dimension.values)): prepared_df = self.prepare_data( @@ -123,27 +137,29 @@ def analyze( dimension_value=dimension_value, ) - analysis_class = analysis_mapping[test.analysis_type] - experiment_analysis = analysis_class( - **test.analysis_config, - target_col=test.metric.components, # todo: add support for ratio and delta method - treatment_col=self.variant_col, - treatment=treatment_variant.name, - ) - ate = experiment_analysis.get_point_estimate(df=prepared_df) p_value = experiment_analysis.get_pvalue(df=prepared_df) std_error = experiment_analysis.get_standard_error( df=prepared_df ) + control_variant_mean = test.metric.get_mean( + prepared_df.query( + f"{self.variant_col}=='{control_variant.name}'" + ) + ) + treatment_variant_mean = test.metric.get_mean( + prepared_df.query( + f"{self.variant_col}=='{treatment_variant.name}'" + ) + ) test_results.append( HypothesisTestResults( metric_alias=test.metric.alias, control_variant_name=control_variant.name, treatment_variant_name=treatment_variant.name, - control_variant_mean=0.5, # todo: add method - treatment_variant_mean=0.6, # todo: add method + control_variant_mean=control_variant_mean, + treatment_variant_mean=treatment_variant_mean, analysis_type=test.analysis_type, ate=ate, ate_ci_lower=0.1, # todo: add method @@ -155,7 +171,7 @@ def analyze( ) ) - return pd.DataFrame([test_result.__dict__ for test_result in test_results]) + return AnalysisPlanResults.from_results(test_results) def prepare_data( self, @@ -206,3 +222,27 @@ def get_treatment_variants(self) -> List[Variant]: A list of treatment variants """ return [variant for variant in self.variants if not variant.is_control] + + @staticmethod + def prepare_analysis_config( + initial_analysis_config: dict, + target_col: str, + treatment_col: str, + treatment: str, + ) -> dict: + """ + Extends the analysis_config provided by the user, by adding or overriding the following keys: + - target_col + - treatment_col + - treatment + + Returns + ------- + dict + The prepared analysis configuration, ready to be ingested by the experiment analysis class + """ + initial_analysis_config["target_col"] = target_col + initial_analysis_config["treatment_col"] = treatment_col + initial_analysis_config["treatment"] = treatment + + return initial_analysis_config diff --git a/cluster_experiments/analysis_results.py b/cluster_experiments/analysis_results.py index e90637a..3bb4117 100644 --- a/cluster_experiments/analysis_results.py +++ b/cluster_experiments/analysis_results.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import List import pandas as pd @@ -58,10 +58,12 @@ class AnalysisPlanResults(pd.DataFrame): """ A class used to represent the results of an Analysis Plan as a pandas DataFrame. + This DataFrame ensures that each row or entry respects the contract defined by the HypothesisTestResults dataclass. + Methods ------- - add_results(results: List[HypothesisTestResults]): - Adds a list of new results to the DataFrame. + from_results(results: List[HypothesisTestResults]): + Creates an AnalysisPlanResults DataFrame from a list of HypothesisTestResults objects. """ def __init__(self, *args, **kwargs): @@ -69,6 +71,8 @@ def __init__(self, *args, **kwargs): "metric_alias", "control_variant_name", "treatment_variant_name", + "dimension_name", + "dimension_value", "control_variant_mean", "treatment_variant_mean", "analysis_type", @@ -77,21 +81,25 @@ def __init__(self, *args, **kwargs): "ate_ci_upper", "p_value", "std_error", - "dimension_name", - "dimension_value", ] super().__init__(*args, columns=columns, **kwargs) - @staticmethod - def add_results(results: List[HypothesisTestResults]) -> pd.DataFrame: + @classmethod + def from_results( + cls, results: List[HypothesisTestResults] + ) -> "AnalysisPlanResults": """ - Adds a list of new results to the DataFrame. + Creates an AnalysisPlanResults DataFrame from a list of HypothesisTestResults objects. Parameters ---------- results : List[HypothesisTestResults] The list of results to be added to the DataFrame + + Returns + ------- + AnalysisPlanResults + A DataFrame containing the results """ - new_data = [result.__dict__ for result in results] - new_df = pd.DataFrame(new_data) - return new_df + data = [asdict(result) for result in results] + return cls(data) diff --git a/cluster_experiments/metric.py b/cluster_experiments/metric.py index 8539840..b326b58 100644 --- a/cluster_experiments/metric.py +++ b/cluster_experiments/metric.py @@ -1,61 +1,219 @@ -from typing import Union +from abc import ABC, abstractmethod +import pandas as pd -class Metric: + +class Metric(ABC): """ - A class used to represent a Metric with an alias and components. + An abstract base class used to represent a Metric with an alias. Attributes ---------- alias : str A string representing the alias of the metric - components : str or tuple - A string or a tuple of strings representing the components of the metric - - Methods - ------- - __init__(self, alias: str, components: tuple): - Initializes the Metric with the provided alias and components. - _validate_inputs(alias: str, components: tuple): - Validates the inputs for the Metric class. """ - def __init__(self, alias: str, components: Union[tuple, str]): + def __init__(self, alias: str): """ Parameters ---------- alias : str The alias of the metric - components : tuple or str - A string or a tuple of strings representing the components of the metric """ - self._validate_inputs(alias, components) + self._validate_alias(alias) self.alias = alias - self.components = components @staticmethod - def _validate_inputs(alias: str, components: tuple): + def _validate_alias(alias: str): """ - Validates the inputs for the Metric class. + Validates the alias input for the Metric class. Parameters ---------- alias : str The alias of the metric - components : tuple or str - A tuple of strings representing the components of the metric Raises ------ TypeError - If the alias is not a string or if components is not a tuple of strings. + If the alias is not a string """ if not isinstance(alias, str): raise TypeError("Metric alias must be a string") - if not isinstance(components, tuple) or not all( - isinstance(comp, str) for comp in components - ): - if not isinstance(components, str): - raise TypeError( - "Metric components must be a string or a tuple of strings" - ) + + @abstractmethod + def get_target_column_from_metric(self) -> str: + """ + Abstract method to return the target column to feed the experiment analysis class, from the metric definition. + + Returns + ------- + str + The target column name + """ + pass + + @abstractmethod + def get_mean(self, df: pd.DataFrame) -> float: + """ + Abstract method to return the mean value of the metric, given a dataframe. + + Returns + ------- + float + The mean value of the metric + """ + pass + + +class SimpleMetric(Metric): + """ + A class used to represent a Simple Metric with an alias and a name. + To be used when the metric is defined at the same level of the data used for the analysis. + + Example + ---------- + In a clustered experiment the participants were randomised based on their country of residence. + The metric of interest is the salary of each participant. If the dataset fed into the analysis is at participant-level, + then a SimpleMetric must be used. However, if the dataset fed into the analysis is at country-level, then a RatioMetric must be used. + + Attributes + ---------- + alias : str + A string representing the alias of the metric + name : str + A string representing the name of the metric + """ + + def __init__(self, alias: str, name: str): + """ + Parameters + ---------- + alias : str + The alias of the metric + name : str + The name of the metric + """ + super().__init__(alias) + self._validate_name(name) + self.name = name + + @staticmethod + def _validate_name(name: str): + """ + Validates the name input for the SimpleMetric class. + + Parameters + ---------- + name : str + The name of the metric + + Raises + ------ + TypeError + If the name is not a string + """ + if not isinstance(name, str): + raise TypeError("SimpleMetric name must be a string") + + def get_target_column_from_metric(self) -> str: + """ + Returns the target column for the SimpleMetric. + + Returns + ------- + str + The name of the metric + """ + return self.name + + def get_mean(self, df: pd.DataFrame) -> float: + """ + Abstract method to return the mean value of the metric, given a dataframe. + + Returns + ------- + float + The mean value of the metric + """ + return df[self.name].mean() + + +class RatioMetric(Metric): + """ + A class used to represent a Ratio Metric with an alias, a numerator name, and a denominator name. + To be used when the metric is defined at a lower level than the data used for the analysis. + + Example + ---------- + In a clustered experiment the participants were randomised based on their country of residence. + The metric of interest is the salary of each participant. If the dataset fed into the analysis is at country-level, + then a RatioMetric must be used: the numerator would be the sum of all salaries in the country, + the denominator would be the number of participants in the country. + + Attributes + ---------- + alias : str + A string representing the alias of the metric + numerator_name : str + A string representing the numerator name of the metric + denominator_name : str + A string representing the denominator name of the metric + """ + + def __init__(self, alias: str, numerator_name: str, denominator_name: str): + """ + Parameters + ---------- + alias : str + The alias of the metric + numerator_name : str + The numerator name of the metric + denominator_name : str + The denominator name of the metric + """ + super().__init__(alias) + self._validate_name(numerator_name) + self._validate_name(denominator_name) + self.numerator_name = numerator_name + self.denominator_name = denominator_name + + @staticmethod + def _validate_name(name: str): + """ + Validates the name input for the RatioMetric class. + + Parameters + ---------- + name : str + The name to validate + + Raises + ------ + TypeError + If the name is not a string + """ + if not isinstance(name, str): + raise TypeError("RatioMetric names must be strings") + + def get_target_column_from_metric(self) -> str: + """ + Returns the target column for the RatioMetric. + + Returns + ------- + str + The numerator name of the metric + """ + return self.numerator_name + + def get_mean(self, df: pd.DataFrame) -> float: + """ + Abstract method to return the mean value of the metric, given a dataframe. + + Returns + ------- + float + The mean value of the metric + """ + return df[self.numerator_name].mean() / df[self.denominator_name].mean() From 62562a33f2ae5fa3cf21682909009e70654b83a5 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sun, 21 Jul 2024 16:38:15 +0200 Subject: [PATCH 12/49] add confidence interval dataclass --- cluster_experiments/experiment_analysis.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cluster_experiments/experiment_analysis.py b/cluster_experiments/experiment_analysis.py index 7068e01..5b2e205 100644 --- a/cluster_experiments/experiment_analysis.py +++ b/cluster_experiments/experiment_analysis.py @@ -1,5 +1,6 @@ import logging from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import Dict, List, Optional import numpy as np @@ -12,6 +13,17 @@ from cluster_experiments.utils import HypothesisEntries +@dataclass +class ConfidenceInterval: + """ + Class to define the structure of a confidence interval. + """ + + lower: float + upper: float + alpha: float + + class ExperimentAnalysis(ABC): """ Abstract class to run the analysis of a given experiment @@ -97,6 +109,23 @@ def analysis_standard_error( """ raise NotImplementedError("Standard error not implemented for this analysis") + def analysis_confidence_interval( + self, + df: pd.DataFrame, + alpha: float, + verbose: bool = False, + ) -> ConfidenceInterval: + """ + Returns the confidence interval of the analysis. Expects treatment to be 0-1 variable + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + raise NotImplementedError( + "Confidence Interval not implemented for this analysis" + ) + def _data_checks(self, df: pd.DataFrame) -> None: """Checks that the data is correct""" if df[self.target_col].isnull().any(): @@ -142,6 +171,19 @@ def get_standard_error(self, df: pd.DataFrame) -> float: self._data_checks(df=df) return self.analysis_standard_error(df) + def get_confidence_interval( + self, df: pd.DataFrame, alpha: float + ) -> ConfidenceInterval: + """Returns the confidence interval of the analysis + + Arguments: + df: dataframe containing the data to analyze + """ + df = df.copy() + df = self._create_binary_treatment(df) + self._data_checks(df=df) + return self.analysis_confidence_interval(df, alpha) + def pvalue_based_on_hypothesis( self, model_result ) -> float: # todo add typehint statsmodels result From 55ee65b1bf8733393c292ad96555e64d78a4269d Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sun, 21 Jul 2024 16:59:12 +0200 Subject: [PATCH 13/49] add confidence interval logic --- cluster_experiments/analysis_plan.py | 17 +++++- cluster_experiments/experiment_analysis.py | 60 ++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index 663c418..f97c92b 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -22,6 +22,10 @@ class AnalysisPlan: A list of HypothesisTest instances variants : List[Variant] A list of Variant instances + variant_col : str + name of the column with the experiment groups + alpha : float + significance level used to construct confidence intervals Methods ------- @@ -36,6 +40,7 @@ def __init__( tests: List[HypothesisTest], variants: List[Variant], variant_col: str = "treatment", + alpha: float = 0.05, ): """ Parameters @@ -46,11 +51,14 @@ def __init__( A list of Variant instances variant_col : str The name of the column containing the variant names. + alpha : float + significance level used to construct confidence intervals """ self._validate_inputs(tests, variants, variant_col) self.tests = tests self.variants = variants self.variant_col = variant_col + self.alpha = alpha @staticmethod def _validate_inputs( @@ -142,6 +150,11 @@ def analyze( std_error = experiment_analysis.get_standard_error( df=prepared_df ) + confidence_interval = ( + experiment_analysis.get_confidence_interval( + df=prepared_df, alpha=self.alpha + ) + ) control_variant_mean = test.metric.get_mean( prepared_df.query( f"{self.variant_col}=='{control_variant.name}'" @@ -162,8 +175,8 @@ def analyze( treatment_variant_mean=treatment_variant_mean, analysis_type=test.analysis_type, ate=ate, - ate_ci_lower=0.1, # todo: add method - ate_ci_upper=0.2, # todo: add method + ate_ci_lower=confidence_interval.lower, + ate_ci_upper=confidence_interval.upper, p_value=p_value, std_error=std_error, dimension_name=dimension.name, diff --git a/cluster_experiments/experiment_analysis.py b/cluster_experiments/experiment_analysis.py index 5b2e205..84fcdec 100644 --- a/cluster_experiments/experiment_analysis.py +++ b/cluster_experiments/experiment_analysis.py @@ -316,6 +316,26 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl results_gee = self.fit_gee(df) return results_gee.bse[self.treatment_col] + def analysis_confidence_interval( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> ConfidenceInterval: + """Returns the standard error of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_gee = self.fit_gee(df) + # Extract the confidence interval for the treatment column + conf_int_df = results_gee.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_gee.summary()) + + # Return the confidence interval + return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + class ClusteredOLSAnalysis(ExperimentAnalysis): """ @@ -407,6 +427,26 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl results_ols = self.fit_ols_clustered(df) return results_ols.bse[self.treatment_col] + def analysis_confidence_interval( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> ConfidenceInterval: + """Returns the standard error of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_ols = self.fit_ols_clustered(df) + # Extract the confidence interval for the treatment column + conf_int_df = results_ols.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_ols.summary()) + + # Return the confidence interval + return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + class TTestClusteredAnalysis(ExperimentAnalysis): """ @@ -683,6 +723,26 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl results_ols = self.fit_ols(df=df) return results_ols.bse[self.treatment_col] + def analysis_confidence_interval( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> ConfidenceInterval: + """Returns the standard error of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_ols = self.fit_ols(df) + # Extract the confidence interval for the treatment column + conf_int_df = results_ols.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_ols.summary()) + + # Return the confidence interval + return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + @classmethod def from_config(cls, config): """Creates an OLSAnalysis object from a PowerConfig object""" From 83455964624df56282538fd7921346ad5cd8a6f3 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sun, 21 Jul 2024 17:02:35 +0200 Subject: [PATCH 14/49] add alpha in results contract --- cluster_experiments/analysis_plan.py | 1 + cluster_experiments/analysis_results.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index f97c92b..695937e 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -181,6 +181,7 @@ def analyze( std_error=std_error, dimension_name=dimension.name, dimension_value=dimension_value, + alpha=self.alpha, ) ) diff --git a/cluster_experiments/analysis_results.py b/cluster_experiments/analysis_results.py index 3bb4117..3303185 100644 --- a/cluster_experiments/analysis_results.py +++ b/cluster_experiments/analysis_results.py @@ -37,6 +37,8 @@ class HypothesisTestResults: The name of the dimension dimension_value : str The value of the dimension + alpha: float + The significance level of the test """ metric_alias: str @@ -52,6 +54,7 @@ class HypothesisTestResults: std_error: float dimension_name: str dimension_value: str + alpha: float class AnalysisPlanResults(pd.DataFrame): @@ -76,6 +79,7 @@ def __init__(self, *args, **kwargs): "control_variant_mean", "treatment_variant_mean", "analysis_type", + "alpha", "ate", "ate_ci_lower", "ate_ci_upper", From 29333559c5f89bc76dde32b26e0c379f1fae1baa Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Sun, 21 Jul 2024 17:36:26 +0200 Subject: [PATCH 15/49] add support for default total dimension --- cluster_experiments/analysis_plan.py | 2 ++ cluster_experiments/dimension.py | 9 +++++++++ cluster_experiments/hypothesis_test.py | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analysis_plan.py index 695937e..79a962f 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analysis_plan.py @@ -201,6 +201,8 @@ def prepare_data( """ prepared_df = data.copy() + prepared_df = prepared_df.assign(total_dimension="total") + prepared_df = prepared_df.query( f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])" ).query(f"{dimension_name} == '{dimension_value}'") diff --git a/cluster_experiments/dimension.py b/cluster_experiments/dimension.py index 1306cbc..159ba61 100644 --- a/cluster_experiments/dimension.py +++ b/cluster_experiments/dimension.py @@ -56,3 +56,12 @@ def _validate_inputs(name: str, values: List[str]): isinstance(val, str) for val in values ): raise TypeError("Dimension values must be a list of strings") + + +class DefaultDimension(Dimension): + """ + A class used to represent a Dimension with a default value representing total, i.e. no slicing. + """ + + def __init__(self): + super().__init__(name="total_dimension", values=["total"]) diff --git a/cluster_experiments/hypothesis_test.py b/cluster_experiments/hypothesis_test.py index 10f2b2e..ea653f5 100644 --- a/cluster_experiments/hypothesis_test.py +++ b/cluster_experiments/hypothesis_test.py @@ -1,6 +1,6 @@ from typing import List, Optional -from cluster_experiments.dimension import Dimension +from cluster_experiments.dimension import DefaultDimension, Dimension from cluster_experiments.metric import Metric @@ -50,7 +50,7 @@ def __init__( self.metric = metric self.analysis_type = analysis_type self.analysis_config = analysis_config or {} - self.dimensions = dimensions or [Dimension(name="total", values=["total"])] + self.dimensions = [DefaultDimension()] + (dimensions or []) @staticmethod def _validate_inputs( From 85e12aa84187e8e50b77b5bb8a53fc793b483264 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:25:49 +0200 Subject: [PATCH 16/49] refactor directory structure into analytics submodule --- cluster_experiments/analytics/__init__.py | 0 cluster_experiments/{ => analytics}/analysis_plan.py | 6 +++--- cluster_experiments/{ => analytics}/analysis_results.py | 0 cluster_experiments/{ => analytics}/dimension.py | 0 cluster_experiments/{ => analytics}/hypothesis_test.py | 4 ++-- cluster_experiments/{ => analytics}/metric.py | 0 cluster_experiments/{ => analytics}/variant.py | 0 7 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 cluster_experiments/analytics/__init__.py rename cluster_experiments/{ => analytics}/analysis_plan.py (98%) rename cluster_experiments/{ => analytics}/analysis_results.py (100%) rename cluster_experiments/{ => analytics}/dimension.py (100%) rename cluster_experiments/{ => analytics}/hypothesis_test.py (96%) rename cluster_experiments/{ => analytics}/metric.py (100%) rename cluster_experiments/{ => analytics}/variant.py (100%) diff --git a/cluster_experiments/analytics/__init__.py b/cluster_experiments/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cluster_experiments/analysis_plan.py b/cluster_experiments/analytics/analysis_plan.py similarity index 98% rename from cluster_experiments/analysis_plan.py rename to cluster_experiments/analytics/analysis_plan.py index 79a962f..e5ba74d 100644 --- a/cluster_experiments/analysis_plan.py +++ b/cluster_experiments/analytics/analysis_plan.py @@ -3,13 +3,13 @@ import pandas as pd from pandas import DataFrame -from cluster_experiments.analysis_results import ( +from cluster_experiments.analytics.analysis_results import ( AnalysisPlanResults, HypothesisTestResults, ) -from cluster_experiments.hypothesis_test import HypothesisTest +from cluster_experiments.analytics.hypothesis_test import HypothesisTest +from cluster_experiments.analytics.variant import Variant from cluster_experiments.power_config import analysis_mapping -from cluster_experiments.variant import Variant class AnalysisPlan: diff --git a/cluster_experiments/analysis_results.py b/cluster_experiments/analytics/analysis_results.py similarity index 100% rename from cluster_experiments/analysis_results.py rename to cluster_experiments/analytics/analysis_results.py diff --git a/cluster_experiments/dimension.py b/cluster_experiments/analytics/dimension.py similarity index 100% rename from cluster_experiments/dimension.py rename to cluster_experiments/analytics/dimension.py diff --git a/cluster_experiments/hypothesis_test.py b/cluster_experiments/analytics/hypothesis_test.py similarity index 96% rename from cluster_experiments/hypothesis_test.py rename to cluster_experiments/analytics/hypothesis_test.py index ea653f5..2d5e6ef 100644 --- a/cluster_experiments/hypothesis_test.py +++ b/cluster_experiments/analytics/hypothesis_test.py @@ -1,7 +1,7 @@ from typing import List, Optional -from cluster_experiments.dimension import DefaultDimension, Dimension -from cluster_experiments.metric import Metric +from cluster_experiments.analytics.dimension import DefaultDimension, Dimension +from cluster_experiments.analytics.metric import Metric class HypothesisTest: diff --git a/cluster_experiments/metric.py b/cluster_experiments/analytics/metric.py similarity index 100% rename from cluster_experiments/metric.py rename to cluster_experiments/analytics/metric.py diff --git a/cluster_experiments/variant.py b/cluster_experiments/analytics/variant.py similarity index 100% rename from cluster_experiments/variant.py rename to cluster_experiments/analytics/variant.py From 423d285961246042ca3acec519fb74a6ea8682f5 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:29:44 +0200 Subject: [PATCH 17/49] edit docstring for analysis plan --- cluster_experiments/analytics/analysis_plan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster_experiments/analytics/analysis_plan.py b/cluster_experiments/analytics/analysis_plan.py index e5ba74d..99b25a7 100644 --- a/cluster_experiments/analytics/analysis_plan.py +++ b/cluster_experiments/analytics/analysis_plan.py @@ -15,6 +15,7 @@ class AnalysisPlan: """ A class used to represent an Analysis Plan with a list of hypothesis tests and a list of variants. + All the hypothesis tests in the same analysis plan will be analysed with the same dataframe, which will need to be passed in the analyze() method. Attributes ---------- From 5ba0e2983e7b740e93964be52e2a01d7dabc80aa Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:34:40 +0200 Subject: [PATCH 18/49] make Variant a dataclass --- cluster_experiments/analytics/variant.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cluster_experiments/analytics/variant.py b/cluster_experiments/analytics/variant.py index ad8f3c1..053b3a0 100644 --- a/cluster_experiments/analytics/variant.py +++ b/cluster_experiments/analytics/variant.py @@ -1,3 +1,7 @@ +from dataclasses import dataclass + + +@dataclass class Variant: """ A class used to represent a Variant with a name and a control flag. @@ -11,24 +15,20 @@ class Variant: Methods ------- - __init__(self, name: str, is_control: bool): - Initializes the Variant with the provided name and control flag. + __post_init__(self): + Validates the inputs after initialization. _validate_inputs(name: str, is_control: bool): Validates the inputs for the Variant class. """ - def __init__(self, name: str, is_control: bool): + name: str + is_control: bool + + def __post_init__(self): """ - Parameters - ---------- - name : str - The name of the variant - is_control : bool - A boolean indicating if the variant is a control variant + Validates the inputs after initialization. """ - self._validate_inputs(name, is_control) - self.name = name - self.is_control = is_control + self._validate_inputs(self.name, self.is_control) @staticmethod def _validate_inputs(name: str, is_control: bool): From d55a56d9a6ae1bd7fd422593dfc4ef719934852d Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:38:21 +0200 Subject: [PATCH 19/49] make Dimension a dataclass --- cluster_experiments/analytics/dimension.py | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cluster_experiments/analytics/dimension.py b/cluster_experiments/analytics/dimension.py index 159ba61..c3ad914 100644 --- a/cluster_experiments/analytics/dimension.py +++ b/cluster_experiments/analytics/dimension.py @@ -1,6 +1,8 @@ +from dataclasses import dataclass from typing import List +@dataclass class Dimension: """ A class used to represent a Dimension with a name and values. @@ -14,24 +16,20 @@ class Dimension: Methods ------- - __init__(self, name: str, values: List[str]): - Initializes the Dimension with the provided name and values. + __post_init__(self): + Validates the inputs after initialization. _validate_inputs(name: str, values: List[str]): Validates the inputs for the Dimension class. """ - def __init__(self, name: str, values: List[str]): + name: str + values: List[str] + + def __post_init__(self): """ - Parameters - ---------- - name : str - The name of the dimension - values : List[str] - A list of strings representing the possible values of the dimension + Validates the inputs after initialization. """ - self._validate_inputs(name, values) - self.name = name - self.values = values + self._validate_inputs(self.name, self.values) @staticmethod def _validate_inputs(name: str, values: List[str]): @@ -58,6 +56,7 @@ def _validate_inputs(name: str, values: List[str]): raise TypeError("Dimension values must be a list of strings") +@dataclass class DefaultDimension(Dimension): """ A class used to represent a Dimension with a default value representing total, i.e. no slicing. From d9241657dcf3d99abf59b6f51670f8346e97a33f Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:48:59 +0200 Subject: [PATCH 20/49] create inference results logic for abstract experiment analysis class --- cluster_experiments/experiment_analysis.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cluster_experiments/experiment_analysis.py b/cluster_experiments/experiment_analysis.py index 84fcdec..c1a952a 100644 --- a/cluster_experiments/experiment_analysis.py +++ b/cluster_experiments/experiment_analysis.py @@ -24,6 +24,18 @@ class ConfidenceInterval: alpha: float +@dataclass +class InferenceResults: + """ + Class to define the structure of complete statistical analysis results. + """ + + ate: float + pvalue = float + std_error: float + conf_int: ConfidenceInterval + + class ExperimentAnalysis(ABC): """ Abstract class to run the analysis of a given experiment @@ -126,6 +138,23 @@ def analysis_confidence_interval( "Confidence Interval not implemented for this analysis" ) + def analysis_inference_results( + self, + df: pd.DataFrame, + alpha: float, + verbose: bool = False, + ) -> InferenceResults: + """ + Returns the InferenceResults object of the analysis. Expects treatment to be 0-1 variable + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + raise NotImplementedError( + "Inference results are not implemented for this analysis" + ) + def _data_checks(self, df: pd.DataFrame) -> None: """Checks that the data is correct""" if df[self.target_col].isnull().any(): @@ -178,12 +207,25 @@ def get_confidence_interval( Arguments: df: dataframe containing the data to analyze + alpha: significance level """ df = df.copy() df = self._create_binary_treatment(df) self._data_checks(df=df) return self.analysis_confidence_interval(df, alpha) + def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults: + """Returns the inference results of the analysis + + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + """ + df = df.copy() + df = self._create_binary_treatment(df) + self._data_checks(df=df) + return self.analysis_inference_results(df, alpha) + def pvalue_based_on_hypothesis( self, model_result ) -> float: # todo add typehint statsmodels result From bda6e0c7c0c35c1651c171e3cefaf0feb7d88af4 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:59:43 +0200 Subject: [PATCH 21/49] add inference results schema and computation method --- cluster_experiments/experiment_analysis.py | 104 ++++++++++++++++++++- 1 file changed, 100 insertions(+), 4 deletions(-) diff --git a/cluster_experiments/experiment_analysis.py b/cluster_experiments/experiment_analysis.py index c1a952a..d5438e8 100644 --- a/cluster_experiments/experiment_analysis.py +++ b/cluster_experiments/experiment_analysis.py @@ -31,7 +31,7 @@ class InferenceResults: """ ate: float - pvalue = float + p_value = float std_error: float conf_int: ConfidenceInterval @@ -361,7 +361,7 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl def analysis_confidence_interval( self, df: pd.DataFrame, alpha: float, verbose: bool = False ) -> ConfidenceInterval: - """Returns the standard error of the analysis + """Returns the confidence interval of the analysis Arguments: df: dataframe containing the data to analyze alpha: significance level @@ -378,6 +378,38 @@ def analysis_confidence_interval( # Return the confidence interval return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + def analysis_inference_results( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> InferenceResults: + """Returns the inference results of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_gee = self.fit_gee(df) + + std_error = results_gee.bse[self.treatment_col] + ate = results_gee.params[self.treatment_col] + p_value = self.pvalue_based_on_hypothesis(results_gee) + + # Extract the confidence interval for the treatment column + conf_int_df = results_gee.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_gee.summary()) + + # Return the confidence interval + return InferenceResults( + ate=ate, + p_value=p_value, + std_error=std_error, + conf_int=ConfidenceInterval( + lower=lower_bound, upper=upper_bound, alpha=alpha + ), + ) + class ClusteredOLSAnalysis(ExperimentAnalysis): """ @@ -472,7 +504,7 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl def analysis_confidence_interval( self, df: pd.DataFrame, alpha: float, verbose: bool = False ) -> ConfidenceInterval: - """Returns the standard error of the analysis + """Returns the confidence interval of the analysis Arguments: df: dataframe containing the data to analyze alpha: significance level @@ -489,6 +521,38 @@ def analysis_confidence_interval( # Return the confidence interval return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + def analysis_inference_results( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> InferenceResults: + """Returns the inference results of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_ols = self.fit_ols_clustered(df) + + std_error = results_ols.bse[self.treatment_col] + ate = results_ols.params[self.treatment_col] + p_value = self.pvalue_based_on_hypothesis(results_ols) + + # Extract the confidence interval for the treatment column + conf_int_df = results_ols.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_ols.summary()) + + # Return the confidence interval + return InferenceResults( + ate=ate, + p_value=p_value, + std_error=std_error, + conf_int=ConfidenceInterval( + lower=lower_bound, upper=upper_bound, alpha=alpha + ), + ) + class TTestClusteredAnalysis(ExperimentAnalysis): """ @@ -768,7 +832,7 @@ def analysis_standard_error(self, df: pd.DataFrame, verbose: bool = False) -> fl def analysis_confidence_interval( self, df: pd.DataFrame, alpha: float, verbose: bool = False ) -> ConfidenceInterval: - """Returns the standard error of the analysis + """Returns the confidence interval of the analysis Arguments: df: dataframe containing the data to analyze alpha: significance level @@ -785,6 +849,38 @@ def analysis_confidence_interval( # Return the confidence interval return ConfidenceInterval(lower=lower_bound, upper=upper_bound, alpha=alpha) + def analysis_inference_results( + self, df: pd.DataFrame, alpha: float, verbose: bool = False + ) -> InferenceResults: + """Returns the inference results of the analysis + Arguments: + df: dataframe containing the data to analyze + alpha: significance level + verbose (Optional): bool, prints the regression summary if True + """ + results_ols = self.fit_ols(df) + + std_error = results_ols.bse[self.treatment_col] + ate = results_ols.params[self.treatment_col] + p_value = self.pvalue_based_on_hypothesis(results_ols) + + # Extract the confidence interval for the treatment column + conf_int_df = results_ols.conf_int(alpha=alpha) + lower_bound, upper_bound = conf_int_df.loc[self.treatment_col] + + if verbose: + print(results_ols.summary()) + + # Return the confidence interval + return InferenceResults( + ate=ate, + p_value=p_value, + std_error=std_error, + conf_int=ConfidenceInterval( + lower=lower_bound, upper=upper_bound, alpha=alpha + ), + ) + @classmethod def from_config(cls, config): """Creates an OLSAnalysis object from a PowerConfig object""" From a7119953ad1740588874518fe8dbbaf1c71d8526 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:05:15 +0200 Subject: [PATCH 22/49] refactor analysis plan to use inference results --- .../analytics/analysis_plan.py | 22 +++++++------------ cluster_experiments/experiment_analysis.py | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cluster_experiments/analytics/analysis_plan.py b/cluster_experiments/analytics/analysis_plan.py index 99b25a7..5838f76 100644 --- a/cluster_experiments/analytics/analysis_plan.py +++ b/cluster_experiments/analytics/analysis_plan.py @@ -146,16 +146,10 @@ def analyze( dimension_value=dimension_value, ) - ate = experiment_analysis.get_point_estimate(df=prepared_df) - p_value = experiment_analysis.get_pvalue(df=prepared_df) - std_error = experiment_analysis.get_standard_error( - df=prepared_df - ) - confidence_interval = ( - experiment_analysis.get_confidence_interval( - df=prepared_df, alpha=self.alpha - ) + inference_results = experiment_analysis.get_inference_results( + df=prepared_df, alpha=self.alpha ) + control_variant_mean = test.metric.get_mean( prepared_df.query( f"{self.variant_col}=='{control_variant.name}'" @@ -175,11 +169,11 @@ def analyze( control_variant_mean=control_variant_mean, treatment_variant_mean=treatment_variant_mean, analysis_type=test.analysis_type, - ate=ate, - ate_ci_lower=confidence_interval.lower, - ate_ci_upper=confidence_interval.upper, - p_value=p_value, - std_error=std_error, + ate=inference_results.ate, + ate_ci_lower=inference_results.conf_int.lower, + ate_ci_upper=inference_results.conf_int.upper, + p_value=inference_results.p_value, + std_error=inference_results.std_error, dimension_name=dimension.name, dimension_value=dimension_value, alpha=self.alpha, diff --git a/cluster_experiments/experiment_analysis.py b/cluster_experiments/experiment_analysis.py index d5438e8..67a21d2 100644 --- a/cluster_experiments/experiment_analysis.py +++ b/cluster_experiments/experiment_analysis.py @@ -31,7 +31,7 @@ class InferenceResults: """ ate: float - p_value = float + p_value: float std_error: float conf_int: ConfidenceInterval From 1da6b84e240545279e7c9a9519a97f77449939a5 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:22:46 +0200 Subject: [PATCH 23/49] move analysis class selection to hypothesis test class --- cluster_experiments/analytics/analysis_plan.py | 10 +--------- cluster_experiments/analytics/hypothesis_test.py | 3 +++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cluster_experiments/analytics/analysis_plan.py b/cluster_experiments/analytics/analysis_plan.py index 5838f76..328701a 100644 --- a/cluster_experiments/analytics/analysis_plan.py +++ b/cluster_experiments/analytics/analysis_plan.py @@ -9,7 +9,6 @@ ) from cluster_experiments.analytics.hypothesis_test import HypothesisTest from cluster_experiments.analytics.variant import Variant -from cluster_experiments.power_config import analysis_mapping class AnalysisPlan: @@ -103,13 +102,6 @@ def analyze( self, exp_data: pd.DataFrame, # , pre_exp_data: Optional[pd.DataFrame], alpha=0.05 ) -> DataFrame: - ... - # add methods to prepare the filtered dataset based on variants and slicers - # add methods to run the analysis for each of the hypothesis tests, given a filtered dataset - # store each row as a HypothesisTestResults object - # wrap all results in an AnalysisPlanResults object - - # ----- # add all kind of checks on the inputs at the beginning using the data structures # todo: ... @@ -121,7 +113,7 @@ def analyze( for test in self.tests: # todo: add cupac handler here - analysis_class = analysis_mapping[test.analysis_type] + analysis_class = test.analysis_class target_col = test.metric.get_target_column_from_metric() for treatment_variant in treatment_variants: diff --git a/cluster_experiments/analytics/hypothesis_test.py b/cluster_experiments/analytics/hypothesis_test.py index 2d5e6ef..515a2f6 100644 --- a/cluster_experiments/analytics/hypothesis_test.py +++ b/cluster_experiments/analytics/hypothesis_test.py @@ -2,6 +2,7 @@ from cluster_experiments.analytics.dimension import DefaultDimension, Dimension from cluster_experiments.analytics.metric import Metric +from cluster_experiments.power_config import analysis_mapping class HypothesisTest: @@ -52,6 +53,8 @@ def __init__( self.analysis_config = analysis_config or {} self.dimensions = [DefaultDimension()] + (dimensions or []) + self.analysis_class = analysis_mapping[self.analysis_type] + @staticmethod def _validate_inputs( metric: Metric, From 00b06d21ff5f6544b9c2ff1f68cc88aa89268d3a Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:14:20 +0200 Subject: [PATCH 24/49] added support for cupac --- .../analytics/analysis_plan.py | 36 ++++++++++++++----- .../analytics/hypothesis_test.py | 24 ++++++++----- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/cluster_experiments/analytics/analysis_plan.py b/cluster_experiments/analytics/analysis_plan.py index 328701a..d6ea0b4 100644 --- a/cluster_experiments/analytics/analysis_plan.py +++ b/cluster_experiments/analytics/analysis_plan.py @@ -1,4 +1,5 @@ -from typing import List +import copy +from typing import List, Optional import pandas as pd from pandas import DataFrame @@ -9,6 +10,7 @@ ) from cluster_experiments.analytics.hypothesis_test import HypothesisTest from cluster_experiments.analytics.variant import Variant +from cluster_experiments.cupac import CupacHandler class AnalysisPlan: @@ -99,8 +101,7 @@ def _validate_inputs( raise ValueError("Variants list cannot be empty") def analyze( - self, - exp_data: pd.DataFrame, # , pre_exp_data: Optional[pd.DataFrame], alpha=0.05 + self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None ) -> DataFrame: # add all kind of checks on the inputs at the beginning using the data structures @@ -112,7 +113,14 @@ def analyze( control_variant: Variant = self.get_control_variant() for test in self.tests: - # todo: add cupac handler here + cupac_covariate_col = None + if test.is_cupac: + cupac_handler = CupacHandler(**test.cupac_config) + exp_data = cupac_handler.add_covariates( + df=exp_data, pre_experiment_df=pre_exp_data + ) + cupac_covariate_col = cupac_handler.cupac_outcome_name + analysis_class = test.analysis_class target_col = test.metric.get_target_column_from_metric() @@ -123,6 +131,7 @@ def analyze( target_col=target_col, treatment_col=self.variant_col, treatment=treatment_variant.name, + cupac_covariate_col=cupac_covariate_col, ) experiment_analysis = analysis_class(**analysis_config_final) @@ -232,6 +241,7 @@ def prepare_analysis_config( target_col: str, treatment_col: str, treatment: str, + cupac_covariate_col: Optional[str] = None, ) -> dict: """ Extends the analysis_config provided by the user, by adding or overriding the following keys: @@ -239,13 +249,23 @@ def prepare_analysis_config( - treatment_col - treatment + Also handles cupac covariate. + Returns ------- dict The prepared analysis configuration, ready to be ingested by the experiment analysis class """ - initial_analysis_config["target_col"] = target_col - initial_analysis_config["treatment_col"] = treatment_col - initial_analysis_config["treatment"] = treatment + new_analysis_config = copy.deepcopy(initial_analysis_config) + + new_analysis_config["target_col"] = target_col + new_analysis_config["treatment_col"] = treatment_col + new_analysis_config["treatment"] = treatment + + if cupac_covariate_col: + covariates = initial_analysis_config.get("covariates", []) + new_analysis_config["covariates"] = list( + set(covariates + [cupac_covariate_col]) + ) - return initial_analysis_config + return new_analysis_config diff --git a/cluster_experiments/analytics/hypothesis_test.py b/cluster_experiments/analytics/hypothesis_test.py index 515a2f6..1954282 100644 --- a/cluster_experiments/analytics/hypothesis_test.py +++ b/cluster_experiments/analytics/hypothesis_test.py @@ -19,13 +19,8 @@ class HypothesisTest: An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances - - Methods - ------- - __init__(self, metric: Metric, analysis: ExperimentAnalysis, analysis_config: Optional[dict] = None, dimensions: Optional[List[Dimension]] = None): - Initializes the HypothesisTest with the provided metric, analysis, and optional analysis configuration and dimensions. - _validate_inputs(metric: Metric, analysis: ExperimentAnalysis, analysis_config: Optional[dict], dimensions: Optional[List[Dimension]]): - Validates the inputs for the HypothesisTest class. + cupac_config : Optional[dict] + An optional dictionary representing the configuration for the cupac model """ def __init__( @@ -34,6 +29,7 @@ def __init__( analysis_type: str, analysis_config: Optional[dict] = None, dimensions: Optional[List[Dimension]] = None, + cupac_config: Optional[dict] = None, ): """ Parameters @@ -46,14 +42,18 @@ def __init__( An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances + cupac_config : Optional[dict] + An optional dictionary representing the configuration for the cupac model """ self._validate_inputs(metric, analysis_type, analysis_config, dimensions) self.metric = metric self.analysis_type = analysis_type self.analysis_config = analysis_config or {} self.dimensions = [DefaultDimension()] + (dimensions or []) + self.cupac_config = cupac_config or {} self.analysis_class = analysis_mapping[self.analysis_type] + self.is_cupac = bool(cupac_config) @staticmethod def _validate_inputs( @@ -61,6 +61,7 @@ def _validate_inputs( analysis_type: str, analysis_config: Optional[dict], dimensions: Optional[List[Dimension]], + cupac_config: Optional[dict] = None, ): """ Validates the inputs for the HypothesisTest class. @@ -75,12 +76,15 @@ def _validate_inputs( An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances + cupac_config : Optional[dict] + An optional dictionary representing the configuration for the cupac model Raises ------ TypeError If metric is not an instance of Metric, if analysis_type is not an instance of string, - if analysis_config is not a dictionary (when provided), or if dimensions is not a list of Dimension instances (when provided). + if analysis_config is not a dictionary (when provided), or if dimensions is not a list of Dimension instances (when provided), + if cupac_config is not a dictionary (when provided) """ if not isinstance(metric, Metric): raise TypeError("Metric must be an instance of Metric") @@ -88,7 +92,9 @@ def _validate_inputs( raise TypeError("Analysis must be a string") # todo: add better check for analysis_type allowed values if analysis_config is not None and not isinstance(analysis_config, dict): - raise TypeError("Analysis_config must be a dictionary if provided") + raise TypeError("analysis_config must be a dictionary if provided") + if cupac_config is not None and not isinstance(analysis_config, dict): + raise TypeError("cupac_config must be a dictionary if provided") if dimensions is not None and ( not isinstance(dimensions, list) or not all(isinstance(dim, Dimension) for dim in dimensions) From 53cc38143887918c97154757d4d82b75550d3c1b Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:25:41 +0200 Subject: [PATCH 25/49] refactor directory name --- .../{analytics => inference}/__init__.py | 0 .../{analytics => inference}/analysis_plan.py | 8 ++-- .../analysis_results.py | 0 .../{analytics => inference}/dimension.py | 0 .../hypothesis_test.py | 4 +- .../{analytics => inference}/metric.py | 47 ++++++------------- .../{analytics => inference}/variant.py | 0 7 files changed, 21 insertions(+), 38 deletions(-) rename cluster_experiments/{analytics => inference}/__init__.py (100%) rename cluster_experiments/{analytics => inference}/analysis_plan.py (98%) rename cluster_experiments/{analytics => inference}/analysis_results.py (100%) rename cluster_experiments/{analytics => inference}/dimension.py (100%) rename cluster_experiments/{analytics => inference}/hypothesis_test.py (97%) rename cluster_experiments/{analytics => inference}/metric.py (83%) rename cluster_experiments/{analytics => inference}/variant.py (100%) diff --git a/cluster_experiments/analytics/__init__.py b/cluster_experiments/inference/__init__.py similarity index 100% rename from cluster_experiments/analytics/__init__.py rename to cluster_experiments/inference/__init__.py diff --git a/cluster_experiments/analytics/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py similarity index 98% rename from cluster_experiments/analytics/analysis_plan.py rename to cluster_experiments/inference/analysis_plan.py index d6ea0b4..631845a 100644 --- a/cluster_experiments/analytics/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -4,13 +4,13 @@ import pandas as pd from pandas import DataFrame -from cluster_experiments.analytics.analysis_results import ( +from cluster_experiments.cupac import CupacHandler +from cluster_experiments.inference.analysis_results import ( AnalysisPlanResults, HypothesisTestResults, ) -from cluster_experiments.analytics.hypothesis_test import HypothesisTest -from cluster_experiments.analytics.variant import Variant -from cluster_experiments.cupac import CupacHandler +from cluster_experiments.inference.hypothesis_test import HypothesisTest +from cluster_experiments.inference.variant import Variant class AnalysisPlan: diff --git a/cluster_experiments/analytics/analysis_results.py b/cluster_experiments/inference/analysis_results.py similarity index 100% rename from cluster_experiments/analytics/analysis_results.py rename to cluster_experiments/inference/analysis_results.py diff --git a/cluster_experiments/analytics/dimension.py b/cluster_experiments/inference/dimension.py similarity index 100% rename from cluster_experiments/analytics/dimension.py rename to cluster_experiments/inference/dimension.py diff --git a/cluster_experiments/analytics/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py similarity index 97% rename from cluster_experiments/analytics/hypothesis_test.py rename to cluster_experiments/inference/hypothesis_test.py index 1954282..badcaf3 100644 --- a/cluster_experiments/analytics/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -1,7 +1,7 @@ from typing import List, Optional -from cluster_experiments.analytics.dimension import DefaultDimension, Dimension -from cluster_experiments.analytics.metric import Metric +from cluster_experiments.inference.dimension import DefaultDimension, Dimension +from cluster_experiments.inference.metric import Metric from cluster_experiments.power_config import analysis_mapping diff --git a/cluster_experiments/analytics/metric.py b/cluster_experiments/inference/metric.py similarity index 83% rename from cluster_experiments/analytics/metric.py rename to cluster_experiments/inference/metric.py index b326b58..22108b8 100644 --- a/cluster_experiments/analytics/metric.py +++ b/cluster_experiments/inference/metric.py @@ -20,25 +20,19 @@ def __init__(self, alias: str): alias : str The alias of the metric """ - self._validate_alias(alias) self.alias = alias + self._validate_alias() - @staticmethod - def _validate_alias(alias: str): + def _validate_alias(self): """ Validates the alias input for the Metric class. - Parameters - ---------- - alias : str - The alias of the metric - Raises ------ TypeError If the alias is not a string """ - if not isinstance(alias, str): + if not isinstance(self.alias, str): raise TypeError("Metric alias must be a string") @abstractmethod @@ -95,25 +89,19 @@ def __init__(self, alias: str, name: str): The name of the metric """ super().__init__(alias) - self._validate_name(name) self.name = name + self._validate_name() - @staticmethod - def _validate_name(name: str): + def _validate_name(self): """ Validates the name input for the SimpleMetric class. - Parameters - ---------- - name : str - The name of the metric - Raises ------ TypeError If the name is not a string """ - if not isinstance(name, str): + if not isinstance(self.name, str): raise TypeError("SimpleMetric name must be a string") def get_target_column_from_metric(self) -> str: @@ -129,7 +117,7 @@ def get_target_column_from_metric(self) -> str: def get_mean(self, df: pd.DataFrame) -> float: """ - Abstract method to return the mean value of the metric, given a dataframe. + Returns the mean value of the metric, given a dataframe. Returns ------- @@ -173,27 +161,22 @@ def __init__(self, alias: str, numerator_name: str, denominator_name: str): The denominator name of the metric """ super().__init__(alias) - self._validate_name(numerator_name) - self._validate_name(denominator_name) self.numerator_name = numerator_name self.denominator_name = denominator_name + self._validate_names() - @staticmethod - def _validate_name(name: str): + def _validate_names(self): """ - Validates the name input for the RatioMetric class. - - Parameters - ---------- - name : str - The name to validate + Validates the numerator and denominator names input for the RatioMetric class. Raises ------ TypeError - If the name is not a string + If the numerator or denominator names are not strings """ - if not isinstance(name, str): + if not isinstance(self.numerator_name, str) or not isinstance( + self.denominator_name, str + ): raise TypeError("RatioMetric names must be strings") def get_target_column_from_metric(self) -> str: @@ -209,7 +192,7 @@ def get_target_column_from_metric(self) -> str: def get_mean(self, df: pd.DataFrame) -> float: """ - Abstract method to return the mean value of the metric, given a dataframe. + Returns the mean value of the metric, given a dataframe. Returns ------- diff --git a/cluster_experiments/analytics/variant.py b/cluster_experiments/inference/variant.py similarity index 100% rename from cluster_experiments/analytics/variant.py rename to cluster_experiments/inference/variant.py From 85f1e61626d6ee151b1533ccce4bbf01c8e58557 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:27:59 +0200 Subject: [PATCH 26/49] change validation method for variant --- cluster_experiments/inference/variant.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/cluster_experiments/inference/variant.py b/cluster_experiments/inference/variant.py index 053b3a0..6542f58 100644 --- a/cluster_experiments/inference/variant.py +++ b/cluster_experiments/inference/variant.py @@ -12,13 +12,6 @@ class Variant: The name of the variant is_control : bool A boolean indicating if the variant is a control variant - - Methods - ------- - __post_init__(self): - Validates the inputs after initialization. - _validate_inputs(name: str, is_control: bool): - Validates the inputs for the Variant class. """ name: str @@ -28,26 +21,18 @@ def __post_init__(self): """ Validates the inputs after initialization. """ - self._validate_inputs(self.name, self.is_control) + self._validate_inputs() - @staticmethod - def _validate_inputs(name: str, is_control: bool): + def _validate_inputs(self): """ Validates the inputs for the Variant class. - Parameters - ---------- - name : str - The name of the variant - is_control : bool - A boolean indicating if the variant is a control variant - Raises ------ TypeError If the name is not a string or if is_control is not a boolean. """ - if not isinstance(name, str): + if not isinstance(self.name, str): raise TypeError("Variant name must be a string") - if not isinstance(is_control, bool): + if not isinstance(self.is_control, bool): raise TypeError("Variant is_control must be a boolean") From fda49f36a778aac7b43eb7719262304faab10726 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:29:53 +0200 Subject: [PATCH 27/49] change validation method for dimension and rename default dimension --- cluster_experiments/inference/dimension.py | 27 +++++----------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/cluster_experiments/inference/dimension.py b/cluster_experiments/inference/dimension.py index c3ad914..2531377 100644 --- a/cluster_experiments/inference/dimension.py +++ b/cluster_experiments/inference/dimension.py @@ -13,13 +13,6 @@ class Dimension: The name of the dimension values : List[str] A list of strings representing the possible values of the dimension - - Methods - ------- - __post_init__(self): - Validates the inputs after initialization. - _validate_inputs(name: str, values: List[str]): - Validates the inputs for the Dimension class. """ name: str @@ -29,29 +22,21 @@ def __post_init__(self): """ Validates the inputs after initialization. """ - self._validate_inputs(self.name, self.values) + self._validate_inputs() - @staticmethod - def _validate_inputs(name: str, values: List[str]): + def _validate_inputs(self): """ Validates the inputs for the Dimension class. - Parameters - ---------- - name : str - The name of the dimension - values : List[str] - A list of strings representing the possible values of the dimension - Raises ------ TypeError If the name is not a string or if values is not a list of strings. """ - if not isinstance(name, str): + if not isinstance(self.name, str): raise TypeError("Dimension name must be a string") - if not isinstance(values, list) or not all( - isinstance(val, str) for val in values + if not isinstance(self.values, list) or not all( + isinstance(val, str) for val in self.values ): raise TypeError("Dimension values must be a list of strings") @@ -63,4 +48,4 @@ class DefaultDimension(Dimension): """ def __init__(self): - super().__init__(name="total_dimension", values=["total"]) + super().__init__(name="__total_dimension", values=["total"]) From ac5cb9afd50a588939b0b6f998791f5c0908c995 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:32:37 +0200 Subject: [PATCH 28/49] change validation method for analysis plan --- .../inference/analysis_plan.py | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 631845a..a10882e 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -56,28 +56,18 @@ def __init__( alpha : float significance level used to construct confidence intervals """ - self._validate_inputs(tests, variants, variant_col) + self.tests = tests self.variants = variants self.variant_col = variant_col self.alpha = alpha - @staticmethod - def _validate_inputs( - tests: List[HypothesisTest], variants: List[Variant], variant_col: str - ): + self._validate_inputs() + + def _validate_inputs(self): """ Validates the inputs for the AnalysisPlan class. - Parameters - ---------- - tests : List[HypothesisTest] - A list of HypothesisTest instances - variants : List[Variant] - A list of Variant instances - variant_col : str - The name of the column containing the variant names. - Raises ------ TypeError @@ -85,19 +75,19 @@ def _validate_inputs( ValueError If tests or variants are empty lists. """ - if not isinstance(tests, list) or not all( - isinstance(test, HypothesisTest) for test in tests + if not isinstance(self.tests, list) or not all( + isinstance(test, HypothesisTest) for test in self.tests ): raise TypeError("Tests must be a list of HypothesisTest instances") - if not isinstance(variants, list) or not all( - isinstance(variant, Variant) for variant in variants + if not isinstance(self.variants, list) or not all( + isinstance(variant, Variant) for variant in self.variants ): raise TypeError("Variants must be a list of Variant instances") - if not isinstance(variant_col, str): + if not isinstance(self.variant_col, str): raise TypeError("Variant_col must be a string") - if not tests: + if not self.tests: raise ValueError("Tests list cannot be empty") - if not variants: + if not self.variants: raise ValueError("Variants list cannot be empty") def analyze( From 54b24ea17a8e5dd95ee59ef88f40a1c55de7a84e Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:37:36 +0200 Subject: [PATCH 29/49] define variant properties --- cluster_experiments/inference/analysis_plan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index a10882e..e99185e 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -195,7 +195,8 @@ def prepare_data( return prepared_df - def get_control_variant(self) -> Variant: + @property + def control_variant(self) -> Variant: """ Returns the control variant from the list of variants. Raises an error if no control variant is found. @@ -214,7 +215,8 @@ def get_control_variant(self) -> Variant: return variant raise ValueError("No control variant found") - def get_treatment_variants(self) -> List[Variant]: + @property + def treatment_variants(self) -> List[Variant]: """ Returns the treatment variants from the list of variants. From 045d4a1c06bf67a7f256cc2508f02ab257f7510a Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:40:42 +0200 Subject: [PATCH 30/49] define variant properties --- .../inference/analysis_plan.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index e99185e..386d831 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -99,8 +99,6 @@ def analyze( # do it before running the computations below test_results = [] - treatment_variants: List[Variant] = self.get_treatment_variants() - control_variant: Variant = self.get_control_variant() for test in self.tests: cupac_covariate_col = None @@ -114,7 +112,7 @@ def analyze( analysis_class = test.analysis_class target_col = test.metric.get_target_column_from_metric() - for treatment_variant in treatment_variants: + for treatment_variant in self.treatment_variants: analysis_config_final = self.prepare_analysis_config( initial_analysis_config=test.analysis_config, @@ -132,7 +130,7 @@ def analyze( data=exp_data, variant_col=self.variant_col, treatment_variant=treatment_variant, - control_variant=control_variant, + control_variant=self.control_variant, dimension_name=dimension.name, dimension_value=dimension_value, ) @@ -143,7 +141,7 @@ def analyze( control_variant_mean = test.metric.get_mean( prepared_df.query( - f"{self.variant_col}=='{control_variant.name}'" + f"{self.variant_col}=='{self.control_variant.name}'" ) ) treatment_variant_mean = test.metric.get_mean( @@ -155,7 +153,7 @@ def analyze( test_results.append( HypothesisTestResults( metric_alias=test.metric.alias, - control_variant_name=control_variant.name, + control_variant_name=self.control_variant.name, treatment_variant_name=treatment_variant.name, control_variant_mean=control_variant_mean, treatment_variant_mean=treatment_variant_mean, @@ -218,14 +216,22 @@ def control_variant(self) -> Variant: @property def treatment_variants(self) -> List[Variant]: """ - Returns the treatment variants from the list of variants. + Returns the treatment variants from the list of variants. Raises an error if no treatment variants are found. Returns ------- List[Variant] A list of treatment variants + + Raises + ------ + ValueError + If no treatment variants are found """ - return [variant for variant in self.variants if not variant.is_control] + treatments = [variant for variant in self.variants if not variant.is_control] + if not treatments: + raise ValueError("No treatment variants found") + return treatments @staticmethod def prepare_analysis_config( From abade822c9113623f3ff4c2a014edb9ded51c074 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:55:20 +0200 Subject: [PATCH 31/49] move inference results to hypothesis test --- .../inference/analysis_plan.py | 4 +-- .../inference/hypothesis_test.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 386d831..e347ebf 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -122,7 +122,7 @@ def analyze( cupac_covariate_col=cupac_covariate_col, ) - experiment_analysis = analysis_class(**analysis_config_final) + analysis_class(**analysis_config_final) for dimension in test.dimensions: for dimension_value in list(set(dimension.values)): @@ -135,7 +135,7 @@ def analyze( dimension_value=dimension_value, ) - inference_results = experiment_analysis.get_inference_results( + inference_results = test.get_inference_results( df=prepared_df, alpha=self.alpha ) diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index badcaf3..dbb33f3 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -1,5 +1,8 @@ from typing import List, Optional +import pandas as pd + +from cluster_experiments.experiment_analysis import InferenceResults from cluster_experiments.inference.dimension import DefaultDimension, Dimension from cluster_experiments.inference.metric import Metric from cluster_experiments.power_config import analysis_mapping @@ -102,3 +105,26 @@ def _validate_inputs( raise TypeError( "Dimensions must be a list of Dimension instances if provided" ) + + def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults: + """ + Performs inference analysis on the provided DataFrame using the analysis class. + + Parameters + ---------- + df : pd.DataFrame + The dataframe containing the data for analysis. + alpha : float + The significance level to be used in the inference analysis. + + Returns + ------- + InferenceResults + The results containing the statistics of the inference procedure. + """ + + inference_results = self.analysis_class.get_inference_results( + df=df, alpha=alpha + ) + + return inference_results From c5f4b16dbf31e8f9c5d8e1fbc989afc09d902d8c Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:55:48 +0200 Subject: [PATCH 32/49] move inference results to hypothesis test --- cluster_experiments/inference/analysis_plan.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index e347ebf..52701e5 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -28,13 +28,6 @@ class AnalysisPlan: name of the column with the experiment groups alpha : float significance level used to construct confidence intervals - - Methods - ------- - __init__(self, tests: List[HypothesisTest], variants: List[Variant]): - Initializes the AnalysisPlan with the provided list of hypothesis tests and variants. - _validate_inputs(tests: List[HypothesisTest], variants: List[Variant]): - Validates the inputs for the AnalysisPlan class. """ def __init__( From c98639d211c9c013ba01c29af6e18d7e2f1d95e5 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:35:03 +0200 Subject: [PATCH 33/49] move cupac logic and config logic from plan to test --- .../inference/analysis_plan.py | 50 ++----------------- .../inference/hypothesis_test.py | 46 ++++++++++++++++- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 52701e5..b79d1f9 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -1,10 +1,8 @@ -import copy from typing import List, Optional import pandas as pd from pandas import DataFrame -from cluster_experiments.cupac import CupacHandler from cluster_experiments.inference.analysis_results import ( AnalysisPlanResults, HypothesisTestResults, @@ -94,29 +92,22 @@ def analyze( test_results = [] for test in self.tests: - cupac_covariate_col = None if test.is_cupac: - cupac_handler = CupacHandler(**test.cupac_config) - exp_data = cupac_handler.add_covariates( + exp_data = test.cupac_handler.add_covariates( df=exp_data, pre_experiment_df=pre_exp_data ) - cupac_covariate_col = cupac_handler.cupac_outcome_name - analysis_class = test.analysis_class target_col = test.metric.get_target_column_from_metric() for treatment_variant in self.treatment_variants: - analysis_config_final = self.prepare_analysis_config( - initial_analysis_config=test.analysis_config, + test._prepare_analysis_config( target_col=target_col, treatment_col=self.variant_col, treatment=treatment_variant.name, - cupac_covariate_col=cupac_covariate_col, + cupac_covariate_col=test.cupac_covariate_col, ) - analysis_class(**analysis_config_final) - for dimension in test.dimensions: for dimension_value in list(set(dimension.values)): prepared_df = self.prepare_data( @@ -225,38 +216,3 @@ def treatment_variants(self) -> List[Variant]: if not treatments: raise ValueError("No treatment variants found") return treatments - - @staticmethod - def prepare_analysis_config( - initial_analysis_config: dict, - target_col: str, - treatment_col: str, - treatment: str, - cupac_covariate_col: Optional[str] = None, - ) -> dict: - """ - Extends the analysis_config provided by the user, by adding or overriding the following keys: - - target_col - - treatment_col - - treatment - - Also handles cupac covariate. - - Returns - ------- - dict - The prepared analysis configuration, ready to be ingested by the experiment analysis class - """ - new_analysis_config = copy.deepcopy(initial_analysis_config) - - new_analysis_config["target_col"] = target_col - new_analysis_config["treatment_col"] = treatment_col - new_analysis_config["treatment"] = treatment - - if cupac_covariate_col: - covariates = initial_analysis_config.get("covariates", []) - new_analysis_config["covariates"] = list( - set(covariates + [cupac_covariate_col]) - ) - - return new_analysis_config diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index dbb33f3..a36edf3 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -1,7 +1,9 @@ +import copy from typing import List, Optional import pandas as pd +from cluster_experiments.cupac import CupacHandler from cluster_experiments.experiment_analysis import InferenceResults from cluster_experiments.inference.dimension import DefaultDimension, Dimension from cluster_experiments.inference.metric import Metric @@ -57,6 +59,13 @@ def __init__( self.analysis_class = analysis_mapping[self.analysis_type] self.is_cupac = bool(cupac_config) + self.cupac_handler = CupacHandler(self.cupac_config) if self.is_cupac else None + self.cupac_covariate_col = ( + self.cupac_handler.cupac_outcome_name if self.is_cupac else None + ) + + self.new_analysis_config = None + self.experiment_analysis = None @staticmethod def _validate_inputs( @@ -123,8 +132,43 @@ def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResu The results containing the statistics of the inference procedure. """ - inference_results = self.analysis_class.get_inference_results( + self.experiment_analysis = self.analysis_class(**self.new_analysis_config) + inference_results = self.experiment_analysis.get_inference_results( df=df, alpha=alpha ) return inference_results + + def _prepare_analysis_config( + self, + target_col: str, + treatment_col: str, + treatment: str, + cupac_covariate_col: Optional[str] = None, + ) -> None: + """ + Extends the analysis_config provided by the user, by adding or overriding the following keys: + - target_col + - treatment_col + - treatment + + Also handles cupac covariate. + + Returns + ------- + dict + The prepared analysis configuration, ready to be ingested by the experiment analysis class + """ + new_analysis_config = copy.deepcopy(self.analysis_config) + + new_analysis_config["target_col"] = target_col + new_analysis_config["treatment_col"] = treatment_col + new_analysis_config["treatment"] = treatment + + if cupac_covariate_col: + covariates = self.analysis_config.get("covariates", []) + new_analysis_config["covariates"] = list( + set(covariates + [cupac_covariate_col]) + ) + + self.new_analysis_config = new_analysis_config From ceda3b08ce35494682b7a2cc38ccc51c47a80326 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:45:51 +0200 Subject: [PATCH 34/49] move prepare data from plan to test --- .../inference/analysis_plan.py | 26 ++----------------- cluster_experiments/inference/dimension.py | 15 +++++++++++ .../inference/hypothesis_test.py | 23 ++++++++++++++++ 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index b79d1f9..c0dea46 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -109,8 +109,8 @@ def analyze( ) for dimension in test.dimensions: - for dimension_value in list(set(dimension.values)): - prepared_df = self.prepare_data( + for dimension_value in dimension.iterate_dimension_values(): + prepared_df = test.prepare_data( data=exp_data, variant_col=self.variant_col, treatment_variant=treatment_variant, @@ -155,28 +155,6 @@ def analyze( return AnalysisPlanResults.from_results(test_results) - def prepare_data( - self, - data: pd.DataFrame, - variant_col: str, - treatment_variant: Variant, - control_variant: Variant, - dimension_name: str, - dimension_value: str, - ) -> pd.DataFrame: - """ - Prepares the data for the experiment analysis pipeline - """ - prepared_df = data.copy() - - prepared_df = prepared_df.assign(total_dimension="total") - - prepared_df = prepared_df.query( - f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])" - ).query(f"{dimension_name} == '{dimension_value}'") - - return prepared_df - @property def control_variant(self) -> Variant: """ diff --git a/cluster_experiments/inference/dimension.py b/cluster_experiments/inference/dimension.py index 2531377..02c5f55 100644 --- a/cluster_experiments/inference/dimension.py +++ b/cluster_experiments/inference/dimension.py @@ -40,6 +40,21 @@ def _validate_inputs(self): ): raise TypeError("Dimension values must be a list of strings") + def iterate_dimension_values(self): + """ + A generator method to yield name and values from the dimension. + + Yields + ------ + Any + A unique value from the dimension. + """ + seen = set() + for value in self.values: + if value not in seen: + seen.add(value) + yield value + @dataclass class DefaultDimension(Dimension): diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index a36edf3..1690b50 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -7,6 +7,7 @@ from cluster_experiments.experiment_analysis import InferenceResults from cluster_experiments.inference.dimension import DefaultDimension, Dimension from cluster_experiments.inference.metric import Metric +from cluster_experiments.inference.variant import Variant from cluster_experiments.power_config import analysis_mapping @@ -172,3 +173,25 @@ def _prepare_analysis_config( ) self.new_analysis_config = new_analysis_config + + @staticmethod + def prepare_data( + data: pd.DataFrame, + variant_col: str, + treatment_variant: Variant, + control_variant: Variant, + dimension_name: str, + dimension_value: str, + ) -> pd.DataFrame: + """ + Prepares the data for the experiment analysis pipeline + """ + prepared_df = data.copy() + + prepared_df = prepared_df.assign(total_dimension="total") + + prepared_df = prepared_df.query( + f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])" + ).query(f"{dimension_name} == '{dimension_value}'") + + return prepared_df From d48b621fab30f937dbeea96d45718b7eb800ab52 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:55:03 +0200 Subject: [PATCH 35/49] fix bug in cupac handler instantiation --- cluster_experiments/inference/hypothesis_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 1690b50..4b35d64 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -60,7 +60,9 @@ def __init__( self.analysis_class = analysis_mapping[self.analysis_type] self.is_cupac = bool(cupac_config) - self.cupac_handler = CupacHandler(self.cupac_config) if self.is_cupac else None + self.cupac_handler = ( + CupacHandler(**self.cupac_config) if self.is_cupac else None + ) self.cupac_covariate_col = ( self.cupac_handler.cupac_outcome_name if self.is_cupac else None ) @@ -188,7 +190,7 @@ def prepare_data( """ prepared_df = data.copy() - prepared_df = prepared_df.assign(total_dimension="total") + prepared_df = prepared_df.assign(__total_dimension="total") prepared_df = prepared_df.query( f"{variant_col}.isin(['{treatment_variant.name}','{control_variant.name}'])" From 3b7cf5b196e9ea977df41d25eb8b6ae5eada9ebd Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:41:37 +0100 Subject: [PATCH 36/49] raise error to handle missing cupac covariate --- cluster_experiments/inference/hypothesis_test.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 4b35d64..1f6b8e1 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -143,11 +143,7 @@ def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResu return inference_results def _prepare_analysis_config( - self, - target_col: str, - treatment_col: str, - treatment: str, - cupac_covariate_col: Optional[str] = None, + self, target_col: str, treatment_col: str, treatment: str ) -> None: """ Extends the analysis_config provided by the user, by adding or overriding the following keys: @@ -168,10 +164,11 @@ def _prepare_analysis_config( new_analysis_config["treatment_col"] = treatment_col new_analysis_config["treatment"] = treatment - if cupac_covariate_col: - covariates = self.analysis_config.get("covariates", []) - new_analysis_config["covariates"] = list( - set(covariates + [cupac_covariate_col]) + covariates = new_analysis_config.get("covariates", []) + + if self.cupac_covariate_col and self.cupac_covariate_col not in covariates: + raise ValueError( + f"You provided a cupac configuration but did not provide the cupac covariate called {self.cupac_covariate_col} in the analysis_config" ) self.new_analysis_config = new_analysis_config From e618629a9e3d251c871bb200c3a02cd3ed18dc65 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:53:56 +0100 Subject: [PATCH 37/49] raise error to handle missing cupac covariate --- cluster_experiments/inference/analysis_plan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index c0dea46..cd868a5 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -105,7 +105,6 @@ def analyze( target_col=target_col, treatment_col=self.variant_col, treatment=treatment_variant.name, - cupac_covariate_col=test.cupac_covariate_col, ) for dimension in test.dimensions: From 42a82974c043f5af4d32cb3ef8c04d684396e7d4 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:59:49 +0100 Subject: [PATCH 38/49] restructure loops for readability --- cluster_experiments/inference/analysis_plan.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index cd868a5..488b577 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -100,15 +100,15 @@ def analyze( target_col = test.metric.get_target_column_from_metric() for treatment_variant in self.treatment_variants: - - test._prepare_analysis_config( - target_col=target_col, - treatment_col=self.variant_col, - treatment=treatment_variant.name, - ) - for dimension in test.dimensions: for dimension_value in dimension.iterate_dimension_values(): + + test._prepare_analysis_config( + target_col=target_col, + treatment_col=self.variant_col, + treatment=treatment_variant.name, + ) + prepared_df = test.prepare_data( data=exp_data, variant_col=self.variant_col, From 4bf5e473bd7583f5f9d57814612e74ca74b1bd55 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:24:54 +0100 Subject: [PATCH 39/49] refactor analysis results data structures --- .../inference/analysis_plan.py | 43 +++-- .../inference/analysis_results.py | 148 +++++++++--------- 2 files changed, 92 insertions(+), 99 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 488b577..ea5f9ad 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -1,11 +1,10 @@ from typing import List, Optional import pandas as pd -from pandas import DataFrame from cluster_experiments.inference.analysis_results import ( AnalysisPlanResults, - HypothesisTestResults, + EmptyAnalysisPlanResults, ) from cluster_experiments.inference.hypothesis_test import HypothesisTest from cluster_experiments.inference.variant import Variant @@ -83,13 +82,13 @@ def _validate_inputs(self): def analyze( self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None - ) -> DataFrame: + ) -> AnalysisPlanResults: # add all kind of checks on the inputs at the beginning using the data structures # todo: ... # do it before running the computations below - test_results = [] + analysis_results = EmptyAnalysisPlanResults() for test in self.tests: if test.is_cupac: @@ -133,26 +132,26 @@ def analyze( ) ) - test_results.append( - HypothesisTestResults( - metric_alias=test.metric.alias, - control_variant_name=self.control_variant.name, - treatment_variant_name=treatment_variant.name, - control_variant_mean=control_variant_mean, - treatment_variant_mean=treatment_variant_mean, - analysis_type=test.analysis_type, - ate=inference_results.ate, - ate_ci_lower=inference_results.conf_int.lower, - ate_ci_upper=inference_results.conf_int.upper, - p_value=inference_results.p_value, - std_error=inference_results.std_error, - dimension_name=dimension.name, - dimension_value=dimension_value, - alpha=self.alpha, - ) + test_results = AnalysisPlanResults( + metric_alias=[test.metric.alias], + control_variant_name=[self.control_variant.name], + treatment_variant_name=[treatment_variant.name], + control_variant_mean=[control_variant_mean], + treatment_variant_mean=[treatment_variant_mean], + analysis_type=[test.analysis_type], + ate=[inference_results.ate], + ate_ci_lower=[inference_results.conf_int.lower], + ate_ci_upper=[inference_results.conf_int.upper], + p_value=[inference_results.p_value], + std_error=[inference_results.std_error], + dimension_name=[dimension.name], + dimension_value=[dimension_value], + alpha=[self.alpha], ) - return AnalysisPlanResults.from_results(test_results) + analysis_results = analysis_results + test_results + + return analysis_results @property def control_variant(self) -> Variant: diff --git a/cluster_experiments/inference/analysis_results.py b/cluster_experiments/inference/analysis_results.py index 3303185..25490cb 100644 --- a/cluster_experiments/inference/analysis_results.py +++ b/cluster_experiments/inference/analysis_results.py @@ -5,105 +5,99 @@ @dataclass -class HypothesisTestResults: +class AnalysisPlanResults: """ - A dataclass used to represent the results of a Hypothesis Test. + A dataclass used to represent the results of the experiment analysis. Attributes ---------- - metric_alias : str + metric_alias : List[str] The alias of the metric used in the test - control_variant_name : str + control_variant_name : List[str] The name of the control variant - treatment_variant_name : str + treatment_variant_name : List[str] The name of the treatment variant - control_variant_mean : float + control_variant_mean : List[float] The mean value of the control variant - treatment_variant_mean : float + treatment_variant_mean : List[float] The mean value of the treatment variant - analysis_type : str + analysis_type : List[str] The type of analysis performed - ate : float + ate : List[float] The average treatment effect - ate_ci_lower : float + ate_ci_lower : List[float] The lower bound of the confidence interval for the ATE - ate_ci_upper : float + ate_ci_upper : List[float] The upper bound of the confidence interval for the ATE - p_value : float + p_value : List[float] The p-value of the test - std_error : float + std_error : List[float] The standard error of the test - dimension_name : str + dimension_name : List[str] The name of the dimension - dimension_value : str + dimension_value : List[str] The value of the dimension - alpha: float + alpha: List[float] The significance level of the test """ - metric_alias: str - control_variant_name: str - treatment_variant_name: str - control_variant_mean: float - treatment_variant_mean: float - analysis_type: str - ate: float - ate_ci_lower: float - ate_ci_upper: float - p_value: float - std_error: float - dimension_name: str - dimension_value: str - alpha: float + metric_alias: List[str] + control_variant_name: List[str] + treatment_variant_name: List[str] + control_variant_mean: List[float] + treatment_variant_mean: List[float] + analysis_type: List[str] + ate: List[float] + ate_ci_lower: List[float] + ate_ci_upper: List[float] + p_value: List[float] + std_error: List[float] + dimension_name: List[str] + dimension_value: List[str] + alpha: List[float] + def __add__(self, other): + if not isinstance(other, AnalysisPlanResults): + return NotImplemented -class AnalysisPlanResults(pd.DataFrame): - """ - A class used to represent the results of an Analysis Plan as a pandas DataFrame. - - This DataFrame ensures that each row or entry respects the contract defined by the HypothesisTestResults dataclass. - - Methods - ------- - from_results(results: List[HypothesisTestResults]): - Creates an AnalysisPlanResults DataFrame from a list of HypothesisTestResults objects. - """ - - def __init__(self, *args, **kwargs): - columns = [ - "metric_alias", - "control_variant_name", - "treatment_variant_name", - "dimension_name", - "dimension_value", - "control_variant_mean", - "treatment_variant_mean", - "analysis_type", - "alpha", - "ate", - "ate_ci_lower", - "ate_ci_upper", - "p_value", - "std_error", - ] - super().__init__(*args, columns=columns, **kwargs) + return AnalysisPlanResults( + metric_alias=self.metric_alias + other.metric_alias, + control_variant_name=self.control_variant_name + other.control_variant_name, + treatment_variant_name=self.treatment_variant_name + + other.treatment_variant_name, + control_variant_mean=self.control_variant_mean + other.control_variant_mean, + treatment_variant_mean=self.treatment_variant_mean + + other.treatment_variant_mean, + analysis_type=self.analysis_type + other.analysis_type, + ate=self.ate + other.ate, + ate_ci_lower=self.ate_ci_lower + other.ate_ci_lower, + ate_ci_upper=self.ate_ci_upper + other.ate_ci_upper, + p_value=self.p_value + other.p_value, + std_error=self.std_error + other.std_error, + dimension_name=self.dimension_name + other.dimension_name, + dimension_value=self.dimension_value + other.dimension_value, + alpha=self.alpha + other.alpha, + ) - @classmethod - def from_results( - cls, results: List[HypothesisTestResults] - ) -> "AnalysisPlanResults": - """ - Creates an AnalysisPlanResults DataFrame from a list of HypothesisTestResults objects. + def to_dataframe(self): + return pd.DataFrame(asdict(self)) - Parameters - ---------- - results : List[HypothesisTestResults] - The list of results to be added to the DataFrame - Returns - ------- - AnalysisPlanResults - A DataFrame containing the results - """ - data = [asdict(result) for result in results] - return cls(data) +class EmptyAnalysisPlanResults(AnalysisPlanResults): + def __init__(self): + super().__init__( + metric_alias=[], + control_variant_name=[], + treatment_variant_name=[], + control_variant_mean=[], + treatment_variant_mean=[], + analysis_type=[], + ate=[], + ate_ci_lower=[], + ate_ci_upper=[], + p_value=[], + std_error=[], + dimension_name=[], + dimension_value=[], + alpha=[], + ) From 98b490c80eaa6a75231dc9989290e9dc1e1ceb02 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:32:14 +0100 Subject: [PATCH 40/49] add data checks in analyse method --- .../inference/analysis_plan.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index ea5f9ad..af03569 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -84,9 +84,7 @@ def analyze( self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None ) -> AnalysisPlanResults: - # add all kind of checks on the inputs at the beginning using the data structures - # todo: ... - # do it before running the computations below + self._validate_data(exp_data, pre_exp_data) analysis_results = EmptyAnalysisPlanResults() @@ -153,6 +151,35 @@ def analyze( return analysis_results + def _validate_data( + self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None + ): + """ + Validates the input dataframes for the analyze method. + + Parameters + ---------- + exp_data : pd.DataFrame + The experimental data + pre_exp_data : Optional[pd.DataFrame] + The pre-experimental data (optional) + + Raises + ------ + ValueError + If exp_data is not a DataFrame or is empty + If pre_exp_data is provided and is not a DataFrame or is empty + """ + if not isinstance(exp_data, pd.DataFrame): + raise ValueError("exp_data must be a pandas DataFrame") + if exp_data.empty: + raise ValueError("exp_data cannot be empty") + if pre_exp_data is not None: + if not isinstance(pre_exp_data, pd.DataFrame): + raise ValueError("pre_exp_data must be a pandas DataFrame if provided") + if pre_exp_data.empty: + raise ValueError("pre_exp_data cannot be empty if provided") + @property def control_variant(self) -> Variant: """ From 83791eda3401a46b7f59227f03f023f02c1c9dfa Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:52:20 +0100 Subject: [PATCH 41/49] add simplified from_metrics interface for analysis plan --- .../inference/analysis_plan.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index af03569..d7c2fdb 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Any, Dict, List, Optional import pandas as pd @@ -6,7 +6,9 @@ AnalysisPlanResults, EmptyAnalysisPlanResults, ) +from cluster_experiments.inference.dimension import Dimension from cluster_experiments.inference.hypothesis_test import HypothesisTest +from cluster_experiments.inference.metric import Metric from cluster_experiments.inference.variant import Variant @@ -219,3 +221,57 @@ def treatment_variants(self) -> List[Variant]: if not treatments: raise ValueError("No treatment variants found") return treatments + + @classmethod + def from_metrics( + cls, + metrics: List[Metric], + variants: List[Variant], + variant_col: str = "treatment", + alpha: float = 0.05, + dimensions: Optional[List[Dimension]] = None, + analysis_type: str = "default", + analysis_config: Optional[Dict[str, Any]] = None, + ) -> "AnalysisPlan": + """ + Creates a simplified AnalysisPlan instance from a list of metrics. It will create HypothesisTest objects under the hood. + This shortcut does not support cupac, and uses the same dimensions, analysis type and analysis config for all metrics. + + Parameters + ---------- + metrics : List[Metric] + A list of Metric instances + variants : List[Variant] + A list of Variant instances + variant_col : str + The name of the column containing the variant names. + alpha : float + Significance level used to construct confidence intervals + dimensions : Optional[List[Dimension]] + A list of Dimension instances (optional) + analysis_type : str + The type of analysis to be conducted (default: "default") + analysis_config : Optional[Dict[str, Any]] + A dictionary containing analysis configuration options (optional) + + Returns + ------- + AnalysisPlan + An instance of AnalysisPlan + """ + tests = [ + HypothesisTest( + metric=metric, + dimensions=dimensions or [], + analysis_type=analysis_type, + analysis_config=analysis_config or {}, + ) + for metric in metrics + ] + + return cls( + tests=tests, + variants=variants, + variant_col=variant_col, + alpha=alpha, + ) From 08943bb99d5e4828ded28e0b165ae2c37b60db07 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:20:12 +0100 Subject: [PATCH 42/49] add verbose logging --- .../inference/analysis_plan.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index d7c2fdb..4945f69 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict, List, Optional import pandas as pd @@ -11,6 +12,10 @@ from cluster_experiments.inference.metric import Metric from cluster_experiments.inference.variant import Variant +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(message)s") +logger = logging.getLogger(__name__) + class AnalysisPlan: """ @@ -83,9 +88,16 @@ def _validate_inputs(self): raise ValueError("Variants list cannot be empty") def analyze( - self, exp_data: pd.DataFrame, pre_exp_data: Optional[pd.DataFrame] = None + self, + exp_data: pd.DataFrame, + pre_exp_data: Optional[pd.DataFrame] = None, + verbose: bool = False, ) -> AnalysisPlanResults: + """ + Method to run the experiment analysis. + """ + # Validate input data at the beginning self._validate_data(exp_data, pre_exp_data) analysis_results = EmptyAnalysisPlanResults() @@ -102,6 +114,14 @@ def analyze( for dimension in test.dimensions: for dimension_value in dimension.iterate_dimension_values(): + if verbose: + logger.info( + f"Metric: {test.metric.alias}, " + f"Treatment: {treatment_variant.name}, " + f"Dimension: {dimension.name}, " + f"Value: {dimension_value}" + ) + test._prepare_analysis_config( target_col=target_col, treatment_col=self.variant_col, From d20e10b0b8f173ffe46a74ac90b43ce22b3a291a Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:11:43 +0200 Subject: [PATCH 43/49] fix logging config bad practise --- cluster_experiments/inference/analysis_plan.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 4945f69..b25eb20 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -12,10 +12,15 @@ from cluster_experiments.inference.metric import Metric from cluster_experiments.inference.variant import Variant -# Configure logging -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(message)s") logger = logging.getLogger(__name__) +# Set up a default handler, but don't configure the root logger +if not logger.hasHandlers(): + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s - %(message)s")) + logger.addHandler(handler) + logger.setLevel(logging.INFO) # This level can be changed by the user + class AnalysisPlan: """ From 05be002009fdd3bc13cd2e67eb48c3fc9ed352f1 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:19:58 +0200 Subject: [PATCH 44/49] make use of class property for metric's target_column --- cluster_experiments/inference/analysis_plan.py | 2 +- cluster_experiments/inference/metric.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index b25eb20..a8d6ac5 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -113,7 +113,7 @@ def analyze( df=exp_data, pre_experiment_df=pre_exp_data ) - target_col = test.metric.get_target_column_from_metric() + target_col = test.metric.target_column for treatment_variant in self.treatment_variants: for dimension in test.dimensions: diff --git a/cluster_experiments/inference/metric.py b/cluster_experiments/inference/metric.py index 22108b8..1666411 100644 --- a/cluster_experiments/inference/metric.py +++ b/cluster_experiments/inference/metric.py @@ -35,10 +35,11 @@ def _validate_alias(self): if not isinstance(self.alias, str): raise TypeError("Metric alias must be a string") + @property @abstractmethod - def get_target_column_from_metric(self) -> str: + def target_column(self) -> str: """ - Abstract method to return the target column to feed the experiment analysis class, from the metric definition. + Abstract property to return the target column to feed the experiment analysis class, from the metric definition. Returns ------- @@ -104,7 +105,8 @@ def _validate_name(self): if not isinstance(self.name, str): raise TypeError("SimpleMetric name must be a string") - def get_target_column_from_metric(self) -> str: + @property + def target_column(self) -> str: """ Returns the target column for the SimpleMetric. @@ -179,7 +181,8 @@ def _validate_names(self): ): raise TypeError("RatioMetric names must be strings") - def get_target_column_from_metric(self) -> str: + @property + def target_column(self) -> str: """ Returns the target column for the RatioMetric. From ccba3011585658af8a5757c182e599c83c587a0a Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:32:21 +0200 Subject: [PATCH 45/49] move add_covariates logic to hypothesis test --- cluster_experiments/inference/analysis_plan.py | 9 ++------- cluster_experiments/inference/hypothesis_test.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index a8d6ac5..d4e8fda 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -108,12 +108,7 @@ def analyze( analysis_results = EmptyAnalysisPlanResults() for test in self.tests: - if test.is_cupac: - exp_data = test.cupac_handler.add_covariates( - df=exp_data, pre_experiment_df=pre_exp_data - ) - - target_col = test.metric.target_column + exp_data = test.add_covariates(exp_data, pre_exp_data) for treatment_variant in self.treatment_variants: for dimension in test.dimensions: @@ -128,7 +123,7 @@ def analyze( ) test._prepare_analysis_config( - target_col=target_col, + target_col=test.metric.target_column, treatment_col=self.variant_col, treatment=treatment_variant.name, ) diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 1f6b8e1..790a370 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -194,3 +194,16 @@ def prepare_data( ).query(f"{dimension_name} == '{dimension_value}'") return prepared_df + + def add_covariates( + self, exp_data: pd.DataFrame, pre_exp_data: pd.DataFrame + ) -> pd.DataFrame: + """ + If the test is a cupac test, adds the covariates to the experimental data. + """ + if self.is_cupac: + exp_data = self.cupac_handler.add_covariates( + df=exp_data, pre_experiment_df=pre_exp_data + ) + + return exp_data From 697c5fe78d8917ac1f82cbd7081ed7a98b982a67 Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:05:36 +0200 Subject: [PATCH 46/49] move get_test_results code to hypothesis test --- .../inference/analysis_plan.py | 49 ++---------- .../inference/hypothesis_test.py | 79 +++++++++++++++++++ 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index d4e8fda..130e0d3 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -122,51 +122,14 @@ def analyze( f"Value: {dimension_value}" ) - test._prepare_analysis_config( - target_col=test.metric.target_column, - treatment_col=self.variant_col, - treatment=treatment_variant.name, - ) - - prepared_df = test.prepare_data( - data=exp_data, - variant_col=self.variant_col, - treatment_variant=treatment_variant, + test_results = test.get_test_results( + exp_data=exp_data, control_variant=self.control_variant, - dimension_name=dimension.name, + treatment_variant=treatment_variant, + variant_col=self.variant_col, + dimension=dimension, dimension_value=dimension_value, - ) - - inference_results = test.get_inference_results( - df=prepared_df, alpha=self.alpha - ) - - control_variant_mean = test.metric.get_mean( - prepared_df.query( - f"{self.variant_col}=='{self.control_variant.name}'" - ) - ) - treatment_variant_mean = test.metric.get_mean( - prepared_df.query( - f"{self.variant_col}=='{treatment_variant.name}'" - ) - ) - - test_results = AnalysisPlanResults( - metric_alias=[test.metric.alias], - control_variant_name=[self.control_variant.name], - treatment_variant_name=[treatment_variant.name], - control_variant_mean=[control_variant_mean], - treatment_variant_mean=[treatment_variant_mean], - analysis_type=[test.analysis_type], - ate=[inference_results.ate], - ate_ci_lower=[inference_results.conf_int.lower], - ate_ci_upper=[inference_results.conf_int.upper], - p_value=[inference_results.p_value], - std_error=[inference_results.std_error], - dimension_name=[dimension.name], - dimension_value=[dimension_value], - alpha=[self.alpha], + alpha=self.alpha, ) analysis_results = analysis_results + test_results diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 790a370..14d5fec 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -5,6 +5,7 @@ from cluster_experiments.cupac import CupacHandler from cluster_experiments.experiment_analysis import InferenceResults +from cluster_experiments.inference.analysis_results import AnalysisPlanResults from cluster_experiments.inference.dimension import DefaultDimension, Dimension from cluster_experiments.inference.metric import Metric from cluster_experiments.inference.variant import Variant @@ -207,3 +208,81 @@ def add_covariates( ) return exp_data + + def get_test_results( + self, + control_variant: Variant, + treatment_variant: Variant, + variant_col: str, + exp_data: pd.DataFrame, + dimension: Dimension, + dimension_value: str, + alpha: float, + ) -> AnalysisPlanResults: + """ + Performs the hypothesis test on the provided data, for the given dimension value. + + Parameters + ---------- + control_variant : Variant + The control variant + treatment_variant : Variant + The treatment variant + variant_col : str + The column name representing the variant + exp_data : pd.DataFrame + The dataframe containing the data for analysis. + dimension : Dimension + The dimension instance + dimension_value : str + The value of the dimension + alpha : float + The significance level to be used in the inference analysis. + + Returns + ------- + AnalysisPlanResults + The results of the hypothesis test + """ + self._prepare_analysis_config( + target_col=self.metric.target_column, + treatment_col=variant_col, + treatment=treatment_variant.name, + ) + + prepared_df = self.prepare_data( + data=exp_data, + variant_col=variant_col, + treatment_variant=treatment_variant, + control_variant=control_variant, + dimension_name=dimension.name, + dimension_value=dimension_value, + ) + + inference_results = self.get_inference_results(df=prepared_df, alpha=alpha) + + control_variant_mean = self.metric.get_mean( + prepared_df.query(f"{variant_col}=='{control_variant.name}'") + ) + treatment_variant_mean = self.metric.get_mean( + prepared_df.query(f"{variant_col}=='{treatment_variant.name}'") + ) + + test_results = AnalysisPlanResults( + metric_alias=[self.metric.alias], + control_variant_name=[control_variant.name], + treatment_variant_name=[treatment_variant.name], + control_variant_mean=[control_variant_mean], + treatment_variant_mean=[treatment_variant_mean], + analysis_type=[self.analysis_type], + ate=[inference_results.ate], + ate_ci_lower=[inference_results.conf_int.lower], + ate_ci_upper=[inference_results.conf_int.upper], + p_value=[inference_results.p_value], + std_error=[inference_results.std_error], + dimension_name=[dimension.name], + dimension_value=[dimension_value], + alpha=[alpha], + ) + + return test_results From 0cc21d3ee88591d1588303b90d369c0a199e558f Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:18:53 +0200 Subject: [PATCH 47/49] make use of default fields for AnalysisPlanResults dataclass --- .../inference/analysis_plan.py | 7 +-- .../inference/analysis_results.py | 50 ++++++------------- 2 files changed, 17 insertions(+), 40 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index 130e0d3..e3a4658 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -3,10 +3,7 @@ import pandas as pd -from cluster_experiments.inference.analysis_results import ( - AnalysisPlanResults, - EmptyAnalysisPlanResults, -) +from cluster_experiments.inference.analysis_results import AnalysisPlanResults from cluster_experiments.inference.dimension import Dimension from cluster_experiments.inference.hypothesis_test import HypothesisTest from cluster_experiments.inference.metric import Metric @@ -105,7 +102,7 @@ def analyze( # Validate input data at the beginning self._validate_data(exp_data, pre_exp_data) - analysis_results = EmptyAnalysisPlanResults() + analysis_results = AnalysisPlanResults() for test in self.tests: exp_data = test.add_covariates(exp_data, pre_exp_data) diff --git a/cluster_experiments/inference/analysis_results.py b/cluster_experiments/inference/analysis_results.py index 25490cb..23a0f17 100644 --- a/cluster_experiments/inference/analysis_results.py +++ b/cluster_experiments/inference/analysis_results.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from typing import List import pandas as pd @@ -41,20 +41,20 @@ class AnalysisPlanResults: The significance level of the test """ - metric_alias: List[str] - control_variant_name: List[str] - treatment_variant_name: List[str] - control_variant_mean: List[float] - treatment_variant_mean: List[float] - analysis_type: List[str] - ate: List[float] - ate_ci_lower: List[float] - ate_ci_upper: List[float] - p_value: List[float] - std_error: List[float] - dimension_name: List[str] - dimension_value: List[str] - alpha: List[float] + metric_alias: List[str] = field(default_factory=lambda: []) + control_variant_name: List[str] = field(default_factory=lambda: []) + treatment_variant_name: List[str] = field(default_factory=lambda: []) + control_variant_mean: List[float] = field(default_factory=lambda: []) + treatment_variant_mean: List[float] = field(default_factory=lambda: []) + analysis_type: List[str] = field(default_factory=lambda: []) + ate: List[float] = field(default_factory=lambda: []) + ate_ci_lower: List[float] = field(default_factory=lambda: []) + ate_ci_upper: List[float] = field(default_factory=lambda: []) + p_value: List[float] = field(default_factory=lambda: []) + std_error: List[float] = field(default_factory=lambda: []) + dimension_name: List[str] = field(default_factory=lambda: []) + dimension_value: List[str] = field(default_factory=lambda: []) + alpha: List[float] = field(default_factory=lambda: []) def __add__(self, other): if not isinstance(other, AnalysisPlanResults): @@ -81,23 +81,3 @@ def __add__(self, other): def to_dataframe(self): return pd.DataFrame(asdict(self)) - - -class EmptyAnalysisPlanResults(AnalysisPlanResults): - def __init__(self): - super().__init__( - metric_alias=[], - control_variant_name=[], - treatment_variant_name=[], - control_variant_mean=[], - treatment_variant_mean=[], - analysis_type=[], - ate=[], - ate_ci_lower=[], - ate_ci_upper=[], - p_value=[], - std_error=[], - dimension_name=[], - dimension_value=[], - alpha=[], - ) From 8d12b88ab49244ad6ef0a98ef04fcfc86ebdb2bf Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:41:18 +0200 Subject: [PATCH 48/49] add check for analysis type allowed values --- cluster_experiments/inference/hypothesis_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 14d5fec..3ce9d1e 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -106,7 +106,10 @@ def _validate_inputs( raise TypeError("Metric must be an instance of Metric") if not isinstance(analysis_type, str): raise TypeError("Analysis must be a string") - # todo: add better check for analysis_type allowed values + if analysis_type not in analysis_mapping: + raise ValueError( + f"Analysis type {analysis_type} not found in analysis_mapping" + ) if analysis_config is not None and not isinstance(analysis_config, dict): raise TypeError("analysis_config must be a dictionary if provided") if cupac_config is not None and not isinstance(analysis_config, dict): From e4f14e63baaa849f58d2293634117f518188332e Mon Sep 17 00:00:00 2001 From: ludovico-lanni <102026076+ludovico-lanni@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:47:30 +0200 Subject: [PATCH 49/49] fix analysis class handling to ensure flexibility and extensibility of the library --- .../inference/analysis_plan.py | 9 +- .../inference/hypothesis_test.py | 90 ++++++++++++++----- 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/cluster_experiments/inference/analysis_plan.py b/cluster_experiments/inference/analysis_plan.py index e3a4658..f3b0899 100644 --- a/cluster_experiments/inference/analysis_plan.py +++ b/cluster_experiments/inference/analysis_plan.py @@ -1,8 +1,9 @@ import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Type import pandas as pd +from cluster_experiments.experiment_analysis import ExperimentAnalysis from cluster_experiments.inference.analysis_results import AnalysisPlanResults from cluster_experiments.inference.dimension import Dimension from cluster_experiments.inference.hypothesis_test import HypothesisTest @@ -212,6 +213,9 @@ def from_metrics( dimensions: Optional[List[Dimension]] = None, analysis_type: str = "default", analysis_config: Optional[Dict[str, Any]] = None, + custom_analysis_type_mapper: Optional[ + dict[str, Type[ExperimentAnalysis]] + ] = None, ) -> "AnalysisPlan": """ Creates a simplified AnalysisPlan instance from a list of metrics. It will create HypothesisTest objects under the hood. @@ -233,6 +237,8 @@ def from_metrics( The type of analysis to be conducted (default: "default") analysis_config : Optional[Dict[str, Any]] A dictionary containing analysis configuration options (optional) + custom_analysis_type_mapper : Optional[dict[str, Type[ExperimentAnalysis]]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes Returns ------- @@ -245,6 +251,7 @@ def from_metrics( dimensions=dimensions or [], analysis_type=analysis_type, analysis_config=analysis_config or {}, + custom_analysis_type_mapper=custom_analysis_type_mapper or {}, ) for metric in metrics ] diff --git a/cluster_experiments/inference/hypothesis_test.py b/cluster_experiments/inference/hypothesis_test.py index 3ce9d1e..cef8687 100644 --- a/cluster_experiments/inference/hypothesis_test.py +++ b/cluster_experiments/inference/hypothesis_test.py @@ -1,10 +1,10 @@ import copy -from typing import List, Optional +from typing import List, Optional, Type import pandas as pd from cluster_experiments.cupac import CupacHandler -from cluster_experiments.experiment_analysis import InferenceResults +from cluster_experiments.experiment_analysis import ExperimentAnalysis, InferenceResults from cluster_experiments.inference.analysis_results import AnalysisPlanResults from cluster_experiments.inference.dimension import DefaultDimension, Dimension from cluster_experiments.inference.metric import Metric @@ -20,14 +20,16 @@ class HypothesisTest: ---------- metric : Metric An instance of the Metric class - analysis : ExperimentAnalysis - An instance of the ExperimentAnalysis class + analysis_type : str + string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided. analysis_config : Optional[dict] An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances cupac_config : Optional[dict] - An optional dictionary representing the configuration for the cupac model + An optional dictionary representing the configuration for the cupac model + custom_analysis_type_mapper : Optional[dict[str, Type[ExperimentAnalysis]]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes """ def __init__( @@ -37,6 +39,9 @@ def __init__( analysis_config: Optional[dict] = None, dimensions: Optional[List[Dimension]] = None, cupac_config: Optional[dict] = None, + custom_analysis_type_mapper: Optional[ + dict[str, Type[ExperimentAnalysis]] + ] = None, ): """ Parameters @@ -44,22 +49,33 @@ def __init__( metric : Metric An instance of the Metric class analysis_type : str - string mapper to an ExperimentAnalysis + string mapping to an ExperimentAnalysis class. Must be either in the built-in analysis_mapping or in the custom_analysis_type_mapper if provided. analysis_config : Optional[dict] An optional dictionary representing the configuration for the analysis dimensions : Optional[List[Dimension]] An optional list of Dimension instances cupac_config : Optional[dict] An optional dictionary representing the configuration for the cupac model + custom_analysis_type_mapper : Optional[dict[str, Type[ExperimentAnalysis]]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes """ - self._validate_inputs(metric, analysis_type, analysis_config, dimensions) + self._validate_inputs( + metric, + analysis_type, + analysis_config, + dimensions, + cupac_config, + custom_analysis_type_mapper, + ) self.metric = metric self.analysis_type = analysis_type self.analysis_config = analysis_config or {} self.dimensions = [DefaultDimension()] + (dimensions or []) self.cupac_config = cupac_config or {} + self.custom_analysis_type_mapper = custom_analysis_type_mapper or {} - self.analysis_class = analysis_mapping[self.analysis_type] + self.analysis_type_mapper = self.custom_analysis_type_mapper or analysis_mapping + self.analysis_class = self.analysis_type_mapper[self.analysis_type] self.is_cupac = bool(cupac_config) self.cupac_handler = ( CupacHandler(**self.cupac_config) if self.is_cupac else None @@ -78,6 +94,9 @@ def _validate_inputs( analysis_config: Optional[dict], dimensions: Optional[List[Dimension]], cupac_config: Optional[dict] = None, + custom_analysis_type_mapper: Optional[ + dict[str, Type[ExperimentAnalysis]] + ] = None, ): """ Validates the inputs for the HypothesisTest class. @@ -94,26 +113,26 @@ def _validate_inputs( An optional list of Dimension instances cupac_config : Optional[dict] An optional dictionary representing the configuration for the cupac model - - Raises - ------ - TypeError - If metric is not an instance of Metric, if analysis_type is not an instance of string, - if analysis_config is not a dictionary (when provided), or if dimensions is not a list of Dimension instances (when provided), - if cupac_config is not a dictionary (when provided) + custom_analysis_type_mapper : Optional[dict[str, ExperimentAnalysis]] + An optional dictionary mapping the names of custom analysis types to the corresponding ExperimentAnalysis classes """ + # Check if metric is a valid Metric instance if not isinstance(metric, Metric): raise TypeError("Metric must be an instance of Metric") + + # Check if analysis_type is a string if not isinstance(analysis_type, str): raise TypeError("Analysis must be a string") - if analysis_type not in analysis_mapping: - raise ValueError( - f"Analysis type {analysis_type} not found in analysis_mapping" - ) + + # Check if analysis_config is a dictionary when provided if analysis_config is not None and not isinstance(analysis_config, dict): raise TypeError("analysis_config must be a dictionary if provided") - if cupac_config is not None and not isinstance(analysis_config, dict): + + # Check if cupac_config is a dictionary when provided + if cupac_config is not None and not isinstance(cupac_config, dict): raise TypeError("cupac_config must be a dictionary if provided") + + # Check if dimensions is a list of Dimension instances when provided if dimensions is not None and ( not isinstance(dimensions, list) or not all(isinstance(dim, Dimension) for dim in dimensions) @@ -122,6 +141,37 @@ def _validate_inputs( "Dimensions must be a list of Dimension instances if provided" ) + # Validate custom_analysis_type_mapper if provided + if custom_analysis_type_mapper: + # Ensure it's a dictionary + if not isinstance(custom_analysis_type_mapper, dict): + raise TypeError( + "custom_analysis_type_mapper must be a dictionary if provided" + ) + + # Ensure all keys are strings and values are ExperimentAnalysis classes + for key, value in custom_analysis_type_mapper.items(): + if not isinstance(key, str): + raise TypeError( + f"Key '{key}' in custom_analysis_type_mapper must be a string" + ) + if not issubclass(value, ExperimentAnalysis): + raise TypeError( + f"Value '{value}' for key '{key}' in custom_analysis_type_mapper must be a subclass of ExperimentAnalysis" + ) + + # Ensure the analysis_type is in the custom mapper if a custom mapper is provided + if analysis_type not in custom_analysis_type_mapper: + raise ValueError( + f"Analysis type '{analysis_type}' not found in the provided custom_analysis_type_mapper" + ) + + # If no custom_analysis_type_mapper, check if analysis_type exists in the default mapping + elif analysis_type not in analysis_mapping: + raise ValueError( + f"Analysis type '{analysis_type}' not found in analysis_mapping" + ) + def get_inference_results(self, df: pd.DataFrame, alpha: float) -> InferenceResults: """ Performs inference analysis on the provided DataFrame using the analysis class.