Skip to content

Commit

Permalink
Add max_price_impact parameter to KellyBettingStrategy (#433)
Browse files Browse the repository at this point in the history 
* WIP

* Added Slippage-based Kelly strategy

* Fixing PR

* Removed file

* Removed another file

* Removed omen corrections

* WIP

* Payouts increased

* Added tests for kelly slippage | PR comments

* WIP

* Fetching bets with more error checking

* Fix examples/monitor/match_bets_with_langfuse_traces.py

* tidy

* Review comment

* Change match_bets_with_langfuse_traces.py to calculate p_yes MSE for each agent

* Removed dotenv

* Print correlation of p_yes mse with profit

* Sorting output file

* Update prediction_market_agent_tooling/deploy/betting_strategy.py

Co-authored-by: Evan Griffiths <[email protected]>

* Update prediction_market_agent_tooling/deploy/betting_strategy.py

Co-authored-by: Peter Jung <[email protected]>

* Avoided price_impact warnings

* Update prediction_market_agent_tooling/deploy/betting_strategy.py

Co-authored-by: Peter Jung <[email protected]>

* Removed load_dotenv

* Tidy

* Update .env.example

Co-authored-by: Peter Jung <[email protected]>

* Implemented PR comments

* Switched back betting strategy to agreed upon impl

* Fixed mypy

* Improved mypy removing ifs

* Final mypy improvements

* Moved test | refactored Yes,No

* Importing logger from loggers and not loguru

* Made minimize_scalar bounds more robust

* Small changes after merge

* Removed incorrect slippage mentions

* Refactored assert from betting stategy into integration test

* Small fix test

---------

Co-authored-by: evangriffiths <[email protected]>
Co-authored-by: Evan Griffiths <[email protected]>
Co-authored-by: Peter Jung <[email protected]>
  • Loading branch information
@gabrielfior @evangriffiths @kongzii
4 people authored Oct 3, 2024
1 parent 9f0a549 commit dff7c35
Show file tree
Hide file tree
Showing 11 changed files with 464 additions and 119 deletions.
72 changes: 71 additions & 1 deletion examples/monitor/match_bets_with_langfuse_traces.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
)
from prediction_market_agent_tooling.markets.data_models import ResolvedBet
from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket
from prediction_market_agent_tooling.tools.httpx_cached_client import HttpxCachedClient
from prediction_market_agent_tooling.tools.langfuse_client_utils import (
ProcessMarketTrace,
ResolvedBetWithTrace,
Expand All @@ -33,6 +34,11 @@ class SimulatedOutcome(BaseModel):
profit: float


class MSEProfit(BaseModel):
p_yes_mse: list[float]
total_profit: list[float]


def get_outcome_for_trace(
strategy: BettingStrategy,
trace: ProcessMarketTrace,
Expand Down Expand Up @@ -93,27 +99,61 @@ def get_outcome_for_trace(
"DeployableThinkThoroughlyProphetResearchAgent": "pma-think-thoroughly-prophet-research",
"DeployableKnownOutcomeAgent": "pma-knownoutcome",
}

agent_pkey_map = {
k: get_private_key_from_gcp_secret(v) for k, v in agent_gcp_secret_map.items()
}

# Define strategies we want to test out
strategies = [
MaxAccuracyBettingStrategy(bet_amount=1),
MaxAccuracyBettingStrategy(bet_amount=2),
MaxAccuracyBettingStrategy(bet_amount=25),
KellyBettingStrategy(max_bet_amount=1),
KellyBettingStrategy(max_bet_amount=2),
KellyBettingStrategy(max_bet_amount=5),
KellyBettingStrategy(max_bet_amount=25),
MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=1),
MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=2),
MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=25),
MaxExpectedValueBettingStrategy(bet_amount=1),
MaxExpectedValueBettingStrategy(bet_amount=2),
MaxExpectedValueBettingStrategy(bet_amount=5),
MaxExpectedValueBettingStrategy(bet_amount=25),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.01),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.05),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.1),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.15),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.2),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.25),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.3),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.4),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.5),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.6),
KellyBettingStrategy(max_bet_amount=2, max_price_impact=0.7),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.1),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.15),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.2),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.3),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.4),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.5),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.6),
KellyBettingStrategy(max_bet_amount=5, max_price_impact=0.7),
KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.1),
KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.2),
KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.3),
KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.5),
KellyBettingStrategy(max_bet_amount=25, max_price_impact=0.7),
]

httpx_client = HttpxCachedClient().get_client()

overall_md = ""

strat_mse_profits: dict[str, MSEProfit] = {}
for strategy in strategies:
strat_mse_profits[repr(strategy)] = MSEProfit(p_yes_mse=[], total_profit=[])

print("# Agent Bet vs Simulated Bet Comparison")
for agent_name, private_key in agent_pkey_map.items():
print(f"\n## {agent_name}\n")
Expand All @@ -126,6 +166,7 @@ def get_outcome_for_trace(
secret_key=api_keys.langfuse_secret_key.get_secret_value(),
public_key=api_keys.langfuse_public_key,
host=api_keys.langfuse_host,
httpx_client=httpx_client,
)

traces = get_traces_for_agent(
Expand Down Expand Up @@ -196,7 +237,18 @@ def get_outcome_for_trace(
)

details.sort(key=lambda x: x["sim_profit"], reverse=True)
pd.DataFrame.from_records(details).to_csv(
f"{agent_name} - {strategy} - all bets.csv", index=False
)

sum_squared_errors = 0.0
for bet_with_trace in bets_with_traces:
bet = bet_with_trace.bet
trace = bet_with_trace.trace
estimated_p_yes = trace.answer.p_yes
actual_answer = float(bet.market_outcome)
sum_squared_errors += (estimated_p_yes - actual_answer) ** 2
p_yes_mse = sum_squared_errors / len(bets_with_traces)
total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces])
total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces])
total_simulated_amount = sum([so.size for so in simulated_outcomes])
Expand All @@ -212,27 +264,45 @@ def get_outcome_for_trace(
"bet_amount": total_bet_amount,
"bet_profit": total_bet_profit,
"roi": roi,
"p_yes mse": p_yes_mse,
# We don't know these for the original run.
"start_balance": None,
"end_balance": None,
}
)
else:
strat_mse_profits[repr(strategy)].p_yes_mse.append(p_yes_mse)
strat_mse_profits[repr(strategy)].total_profit.append(
total_simulated_profit
)

simulations.append(
{
"strategy": repr(strategy),
"bet_amount": total_simulated_amount,
"bet_profit": total_simulated_profit,
"roi": simulated_roi,
"p_yes mse": p_yes_mse,
"start_balance": starting_balance,
"end_balance": agent_balance,
}
)

simulations_df = pd.DataFrame.from_records(simulations)
simulations_df.sort_values(by="bet_profit", ascending=False, inplace=True)
overall_md += (
f"\n\n## {agent_name}\n\n{len(bets_with_traces)} bets\n\n"
+ pd.DataFrame.from_records(simulations).to_markdown(index=False)
+ simulations_df.to_markdown(index=False)
)
# export details per agent
pd.DataFrame.from_records(details).to_csv(f"{agent_name}_details.csv")

print(f"Correlation between p_yes mse and total profit:")
for strategy_name, mse_profit in strat_mse_profits.items():
mse = mse_profit.p_yes_mse
profit = mse_profit.total_profit
correlation = pd.Series(mse).corr(pd.Series(profit))
print(f"{strategy_name}: {correlation=}")

with open("match_bets_with_langfuse_traces_overall.md", "w") as overall_f:
overall_f.write(overall_md)
Loading

0 comments on commit dff7c35

Please sign in to comment.