Add max_price_impact parameter to KellyBettingStrategy

examples/monitor/match_bets_with_langfuse_traces.py

@@ -17,6 +17,7 @@
 )
 from prediction_market_agent_tooling.markets.data_models import ResolvedBet
 from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket
+from prediction_market_agent_tooling.tools.httpx_cached_client import HttpxCachedClient
 from prediction_market_agent_tooling.tools.langfuse_client_utils import (
     ProcessMarketTrace,
     ResolvedBetWithTrace,
@@ -33,6 +34,11 @@ class SimulatedOutcome(BaseModel):
     profit: float
 
 
+class MSEProfit(BaseModel):
+    p_yes_mse: list[float]
+    total_profit: list[float]
+
+
 def get_outcome_for_trace(
     strategy: BettingStrategy,
     trace: ProcessMarketTrace,
@@ -93,27 +99,61 @@ def get_outcome_for_trace(
         "DeployableThinkThoroughlyProphetResearchAgent": "pma-think-thoroughly-prophet-research",
         "DeployableKnownOutcomeAgent": "pma-knownoutcome",
     }
+
     agent_pkey_map = {
         k: get_private_key_from_gcp_secret(v) for k, v in agent_gcp_secret_map.items()
     }
+
     # Define strategies we want to test out
     strategies = [
         MaxAccuracyBettingStrategy(bet_amount=1),
         MaxAccuracyBettingStrategy(bet_amount=2),
         MaxAccuracyBettingStrategy(bet_amount=25),
         KellyBettingStrategy(max_bet_amount=1),
         KellyBettingStrategy(max_bet_amount=2),
+        KellyBettingStrategy(max_bet_amount=5),
         KellyBettingStrategy(max_bet_amount=25),
         MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=1),
         MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=2),
         MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=25),
         MaxExpectedValueBettingStrategy(bet_amount=1),
         MaxExpectedValueBettingStrategy(bet_amount=2),
+        MaxExpectedValueBettingStrategy(bet_amount=5),
         MaxExpectedValueBettingStrategy(bet_amount=25),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.01),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.05),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.1),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.15),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.2),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.25),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.3),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.4),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.5),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.6),
+        KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.7),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.1),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.15),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.2),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.3),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.4),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.5),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.6),
+        KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.7),
+        KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.1),
+        KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.2),
+        KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.3),
+        KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.5),
+        KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.7),
     ]
 
+    httpx_client = HttpxCachedClient().get_client()
+
     overall_md = ""
 
+    strat_mse_profits: dict[str, MSEProfit] = {}
+    for strategy in strategies:
+        strat_mse_profits[repr(strategy)] = MSEProfit(p_yes_mse=[], total_profit=[])
+
     print("# Agent Bet vs Simulated Bet Comparison")
     for agent_name, private_key in agent_pkey_map.items():
         print(f"\n## {agent_name}\n")
@@ -126,6 +166,7 @@ def get_outcome_for_trace(
             secret_key=api_keys.langfuse_secret_key.get_secret_value(),
             public_key=api_keys.langfuse_public_key,
             host=api_keys.langfuse_host,
+            httpx_client=httpx_client,
         )
 
         traces = get_traces_for_agent(
@@ -196,7 +237,18 @@ def get_outcome_for_trace(
                 )
 
             details.sort(key=lambda x: x["sim_profit"], reverse=True)
+            pd.DataFrame.from_records(details).to_csv(
+                f"{agent_name} - {strategy} - all bets.csv", index=False
+            )
 
+            sum_squared_errors = 0.0
+            for bet_with_trace in bets_with_traces:
+                bet = bet_with_trace.bet
+                trace = bet_with_trace.trace
+                estimated_p_yes = trace.answer.p_yes
+                actual_answer = float(bet.market_outcome)
+                sum_squared_errors += (estimated_p_yes - actual_answer) ** 2
+            p_yes_mse = sum_squared_errors / len(bets_with_traces)
             total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces])
             total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces])
             total_simulated_amount = sum([so.size for so in simulated_outcomes])
@@ -212,27 +264,45 @@ def get_outcome_for_trace(
                         "bet_amount": total_bet_amount,
                         "bet_profit": total_bet_profit,
                         "roi": roi,
+                        "p_yes mse": p_yes_mse,
                         # We don't know these for the original run.
                         "start_balance": None,
                         "end_balance": None,
                     }
                 )
+            else:
+                strat_mse_profits[repr(strategy)].p_yes_mse.append(p_yes_mse)
+                strat_mse_profits[repr(strategy)].total_profit.append(
+                    total_simulated_profit
+                )
 
             simulations.append(
                 {
                     "strategy": repr(strategy),
                     "bet_amount": total_simulated_amount,
                     "bet_profit": total_simulated_profit,
                     "roi": simulated_roi,
+                    "p_yes mse": p_yes_mse,
                     "start_balance": starting_balance,
                     "end_balance": agent_balance,
                 }
             )
 
+        simulations_df = pd.DataFrame.from_records(simulations)
+        simulations_df.sort_values(by="bet_profit", ascending=False, inplace=True)
         overall_md += (
             f"\n\n## {agent_name}\n\n{len(bets_with_traces)} bets\n\n"
-            + pd.DataFrame.from_records(simulations).to_markdown(index=False)
+            + simulations_df.to_markdown(index=False)
         )
+        # export details per agent
+        pd.DataFrame.from_records(details).to_csv(f"{agent_name}_details.csv")
+
+    print(f"Correlation between p_yes mse and total profit:")
+    for strategy_name, mse_profit in strat_mse_profits.items():
+        mse = mse_profit.p_yes_mse
+        profit = mse_profit.total_profit
+        correlation = pd.Series(mse).corr(pd.Series(profit))
+        print(f"{strategy_name}: {correlation=}")
 
     with open("match_bets_with_langfuse_traces_overall.md", "w") as overall_f:
         overall_f.write(overall_md)