diff --git a/.gitignore b/.gitignore index 68bc17f9..1ff55bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,7 @@ celerybeat.pid # Environments .env +.env.monitor .venv env/ venv/ diff --git a/examples/monitor/monitor.py b/examples/monitor/monitor.py index 1de74121..5fc37504 100644 --- a/examples/monitor/monitor.py +++ b/examples/monitor/monitor.py @@ -1,21 +1,8 @@ -from datetime import datetime, timedelta -from zoneinfo import ZoneInfo - import streamlit as st -from prediction_market_agent_tooling.markets.manifold import get_authenticated_user -from prediction_market_agent_tooling.monitor.markets.manifold import ( - DeployedManifoldAgent, -) -from prediction_market_agent_tooling.monitor.monitor import monitor_agent +from prediction_market_agent_tooling.monitor.monitor_app import monitor_app if __name__ == "__main__": - start_time = datetime.now() - timedelta(weeks=1) - agent = DeployedManifoldAgent( - name="foo", - start_time=start_time.astimezone(ZoneInfo("UTC")), - manifold_user_id=get_authenticated_user().id, - ) st.set_page_config(layout="wide") # Best viewed with a wide screen - st.title(f"Monitoring Agent: '{agent.name}'") - monitor_agent(agent) + st.title(f"Monitoring") + monitor_app() diff --git a/poetry.lock b/poetry.lock index 7405c650..9fe03995 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3410,6 +3410,17 @@ dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2 doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +[[package]] +name = "types-pytz" +version = "2024.1.0.20240203" +description = "Typing stubs for pytz" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-pytz-2024.1.0.20240203.tar.gz", hash = "sha256:c93751ee20dfc6e054a0148f8f5227b9a00b79c90a4d3c9f464711a73179c89e"}, + {file = "types_pytz-2024.1.0.20240203-py3-none-any.whl", hash = "sha256:9679eef0365db3af91ef7722c199dbb75ee5c1b67e3c4dd7bfbeb1b8a71c21a3"}, +] + [[package]] name = "types-requests" version = "2.31.0.20240125" @@ -3812,4 +3823,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "9dd189d8c434efbdf54f3d66911e2db682e9fed90f3e3aef0c81e6810ddf8532" +content-hash = "a865a2a39c0916578171935f6c199f8500aa862940dacd0614bd610da3c8a1a9" diff --git a/prediction_market_agent_tooling/benchmark/utils.py b/prediction_market_agent_tooling/benchmark/utils.py index 344d2fd0..8563da4a 100644 --- a/prediction_market_agent_tooling/benchmark/utils.py +++ b/prediction_market_agent_tooling/benchmark/utils.py @@ -1,10 +1,14 @@ import json import typing as t +from datetime import datetime from enum import Enum +import pytz import requests from pydantic import BaseModel, validator +MANIFOLD_API_LIMIT = 1000 # Manifold will only return up to 1000 markets + class EvaluatedQuestion(BaseModel): question: str @@ -23,6 +27,7 @@ class Market(BaseModel): p_yes: float volume: float is_resolved: bool + created_time: datetime resolution: str | None = None outcomePrices: list[float] | None = None @@ -34,6 +39,12 @@ def _validate_outcome_prices(cls, value: list[float] | None) -> list[float] | No raise ValueError("outcomePrices must have exactly 2 elements.") return value + @validator("created_time") + def _validate_created_time(cls, value: datetime) -> datetime: + if value.tzinfo is None: + value = value.replace(tzinfo=pytz.UTC) + return value + @property def p_no(self) -> float: return 1 - self.p_yes @@ -103,22 +114,24 @@ def save(self, path: str) -> None: @staticmethod def load(path: str) -> "PredictionsCache": with open(path, "r") as f: - return PredictionsCache.parse_obj(json.load(f)) + return PredictionsCache.model_validate(json.load(f)) def get_manifold_markets( - number: int = 100, - excluded_questions: t.List[str] = [], + limit: int = 100, + offset: int = 0, filter_: t.Literal[ "open", "closed", "resolved", "closing-this-month", "closing-next-month" ] = "open", + sort: t.Literal["liquidity", "score", "newest"] = "liquidity", ) -> t.List[Market]: url = "https://api.manifold.markets/v0/search-markets" params = { "term": "", - "sort": "liquidity", + "sort": sort, "filter": filter_, - "limit": f"{number + len(excluded_questions)}", + "limit": f"{limit}", + "offset": offset, "contractType": "BINARY", # TODO support CATEGORICAL markets } response = requests.get(url, params=params) @@ -132,27 +145,85 @@ def get_manifold_markets( fields_map = { "probability": "p_yes", "isResolved": "is_resolved", + "createdTime": "created_time", } def _map_fields(old: dict[str, str], mapping: dict[str, str]) -> dict[str, str]: return {mapping.get(k, k): v for k, v in old.items()} - markets = [Market.parse_obj(_map_fields(m, fields_map)) for m in markets_json] + markets = [Market.model_validate(_map_fields(m, fields_map)) for m in markets_json] + + return markets + + +def get_manifold_markets_paged( + number: int = 100, + filter_: t.Literal[ + "open", "closed", "resolved", "closing-this-month", "closing-next-month" + ] = "open", + sort: t.Literal["liquidity", "score", "newest"] = "liquidity", + starting_offset: int = 0, + excluded_questions: set[str] | None = None, +) -> t.List[Market]: + markets: list[Market] = [] + + offset = starting_offset + while len(markets) < number: + new_markets = get_manifold_markets( + limit=min(MANIFOLD_API_LIMIT, number - len(markets)), + offset=offset, + filter_=filter_, + sort=sort, + ) + if not new_markets: + break + markets.extend( + market + for market in new_markets + if not excluded_questions or market.question not in excluded_questions + ) + offset += len(new_markets) + + return markets - # Filter out markets with excluded questions - markets = [m for m in markets if m.question not in excluded_questions] - return markets[:number] +def get_manifold_markets_dated( + oldest_date: datetime, + filter_: t.Literal[ + "open", "closed", "resolved", "closing-this-month", "closing-next-month" + ] = "open", + excluded_questions: set[str] | None = None, +) -> t.List[Market]: + markets: list[Market] = [] + + offset = 0 + while True: + new_markets = get_manifold_markets( + limit=MANIFOLD_API_LIMIT, + offset=offset, + filter_=filter_, + sort="newest", # Enforce sorting by newest, because there aren't date filters on the API. + ) + if not new_markets: + break + for market in new_markets: + if market.created_time < oldest_date: + return markets + if not excluded_questions or market.question not in excluded_questions: + markets.append(market) + offset += 1 + + return markets def get_polymarket_markets( - number: int = 100, - excluded_questions: t.List[str] = [], + limit: int = 100, active: bool | None = True, closed: bool | None = False, + excluded_questions: set[str] | None = None, ) -> t.List[Market]: params: dict[str, str | int] = { - "_limit": number + len(excluded_questions), + "_limit": limit, } if active is not None: params["active"] = "true" if active else "false" @@ -167,8 +238,7 @@ def get_polymarket_markets( if m_json["outcomes"] != ["Yes", "No"]: continue - if m_json["question"] in excluded_questions: - print(f"Skipping market with 'excluded question': {m_json['question']}") + if excluded_questions and m_json["question"] in excluded_questions: continue markets.append( @@ -178,6 +248,7 @@ def get_polymarket_markets( p_yes=m_json["outcomePrices"][ 0 ], # For binary markets on Polymarket, the first outcome is "Yes" and outcomePrices are equal to probabilities. + created_time=m_json["created_at"], outcomePrices=m_json["outcomePrices"], volume=m_json["volume"], is_resolved=False, @@ -190,15 +261,15 @@ def get_polymarket_markets( def get_markets( number: int, source: MarketSource, - excluded_questions: t.List[str] = [], + excluded_questions: set[str] | None = None, ) -> t.List[Market]: if source == MarketSource.MANIFOLD: - return get_manifold_markets( + return get_manifold_markets_paged( number=number, excluded_questions=excluded_questions ) elif source == MarketSource.POLYMARKET: return get_polymarket_markets( - number=number, excluded_questions=excluded_questions + limit=number, excluded_questions=excluded_questions ) else: raise ValueError(f"Unknown market source: {source}") diff --git a/prediction_market_agent_tooling/markets/data_models.py b/prediction_market_agent_tooling/markets/data_models.py index 68cc5e4f..6cb30db1 100644 --- a/prediction_market_agent_tooling/markets/data_models.py +++ b/prediction_market_agent_tooling/markets/data_models.py @@ -156,7 +156,7 @@ class ManifoldMarket(BaseModel): isResolved: bool resolution: t.Optional[str] = None resolutionTime: t.Optional[datetime] = None - lastBetTime: datetime + lastBetTime: t.Optional[datetime] = None lastCommentTime: t.Optional[datetime] = None lastUpdatedTime: datetime mechanism: str diff --git a/prediction_market_agent_tooling/markets/manifold.py b/prediction_market_agent_tooling/markets/manifold.py index c5783a21..71df8c0d 100644 --- a/prediction_market_agent_tooling/markets/manifold.py +++ b/prediction_market_agent_tooling/markets/manifold.py @@ -81,10 +81,10 @@ def place_bet(amount: Mana, market_id: str, outcome: bool) -> None: ) -def get_authenticated_user() -> ManifoldUser: +def get_authenticated_user(api_key: str) -> ManifoldUser: url = "https://api.manifold.markets/v0/me" headers = { - "Authorization": f"Key {APIKeys().manifold_api_key}", + "Authorization": f"Key {api_key}", "Content-Type": "application/json", } response = requests.get(url, headers=headers) diff --git a/prediction_market_agent_tooling/monitor/monitor.py b/prediction_market_agent_tooling/monitor/monitor.py index 1adeaa33..03a76786 100644 --- a/prediction_market_agent_tooling/monitor/monitor.py +++ b/prediction_market_agent_tooling/monitor/monitor.py @@ -1,12 +1,16 @@ import typing as t from datetime import datetime +from itertools import groupby import altair as alt +import numpy as np import pandas as pd import streamlit as st from pydantic import BaseModel +from prediction_market_agent_tooling.benchmark.utils import Market from prediction_market_agent_tooling.markets.data_models import ResolvedBet +from prediction_market_agent_tooling.tools.utils import should_not_happen class DeployedAgent(BaseModel): @@ -20,6 +24,9 @@ def get_resolved_bets(self) -> list[ResolvedBet]: def monitor_agent(agent: DeployedAgent) -> None: agent_bets = agent.get_resolved_bets() + if not agent_bets: + st.warning(f"No resolved bets found for {agent.name}.") + return bets_info = { "Market Question": [bet.market_question for bet in agent_bets], "Bet Amount": [bet.amount.amount for bet in agent_bets], @@ -47,7 +54,6 @@ def monitor_agent(agent: DeployedAgent) -> None: profit_df.groupby("Date")["Cumulative Profit"].sum().cumsum().reset_index() ) profit_df["Cumulative Profit"] = profit_df["Cumulative Profit"].astype(float) - st.empty() st.altair_chart( alt.Chart(profit_df) .mark_line() @@ -60,6 +66,80 @@ def monitor_agent(agent: DeployedAgent) -> None: ) # Table of resolved bets - st.empty() st.subheader("Resolved Bet History") st.table(bets_df) + + +def monitor_market(open_markets: list[Market], resolved_markets: list[Market]) -> None: + date_to_open_yes_proportion = { + d: np.mean([int(m.p_yes > 0.5) for m in markets]) + for d, markets in groupby(open_markets, lambda x: x.created_time.date()) + } + date_to_resolved_yes_proportion = { + d: np.mean( + [ + ( + 1 + if m.resolution == "YES" + else ( + 0 + if m.resolution == "NO" + else should_not_happen(f"Unexpected resolution: {m.resolution}") + ) + ) + for m in markets + ] + ) + for d, markets in groupby(resolved_markets, lambda x: x.created_time.date()) + } + + df_open = pd.DataFrame( + date_to_open_yes_proportion.items(), columns=["date", "open_proportion"] + ) + df_open["open_label"] = "Open's yes proportion" + df_resolved = pd.DataFrame( + date_to_resolved_yes_proportion.items(), columns=["date", "resolved_proportion"] + ) + df_resolved["resolved_label"] = "Resolved's yes proportion" + + df = pd.merge(df_open, df_resolved, on="date") + + open_chart = ( + alt.Chart(df) + .mark_line() + .encode(x="date:T", y="open_proportion:Q", color="open_label:N") + ) + + resolved_chart = ( + alt.Chart(df) + .mark_line() + .encode(x="date:T", y="resolved_proportion:Q", color="resolved_label:N") + ) + + st.altair_chart( + alt.layer(open_chart, resolved_chart).interactive(), # type: ignore # Doesn't expect `LayerChart`, but `Chart`, yet it works. + use_container_width=True, + ) + + all_open_markets_yes_mean = np.mean([int(m.p_yes > 0.5) for m in open_markets]) + all_resolved_markets_yes_mean = np.mean( + [ + ( + 1 + if m.resolution == "YES" + else ( + 0 + if m.resolution == "NO" + else should_not_happen(f"Unexpected resolution: {m.resolution}") + ) + ) + for m in resolved_markets + ] + ) + st.markdown( + f"Total number of open markets {len(open_markets)} and resolved markets {len(resolved_markets)}" + "\n\n" + f"Mean proportion of 'YES' in open markets: {all_open_markets_yes_mean:.2f}" + "\n\n" + f"Mean proportion of 'YES' in resolved markets: {all_resolved_markets_yes_mean:.2f}" + ) diff --git a/prediction_market_agent_tooling/monitor/monitor_app.py b/prediction_market_agent_tooling/monitor/monitor_app.py new file mode 100644 index 00000000..1ddd037f --- /dev/null +++ b/prediction_market_agent_tooling/monitor/monitor_app.py @@ -0,0 +1,75 @@ +import typing as t +from datetime import date, datetime, timedelta + +import pytz +import streamlit as st +from pydantic_settings import BaseSettings, SettingsConfigDict + +from prediction_market_agent_tooling.benchmark.utils import get_manifold_markets_dated +from prediction_market_agent_tooling.markets.manifold import get_authenticated_user +from prediction_market_agent_tooling.markets.markets import MarketType +from prediction_market_agent_tooling.monitor.markets.manifold import ( + DeployedManifoldAgent, +) +from prediction_market_agent_tooling.monitor.monitor import ( + monitor_agent, + monitor_market, +) +from prediction_market_agent_tooling.tools.utils import check_not_none + + +class MonitorSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env.monitor", env_file_encoding="utf-8", extra="ignore" + ) + + MANIFOLD_API_KEYS: list[str] = [] + PAST_N_WEEKS: int = 1 + + +def monitor_app() -> None: + settings = MonitorSettings() + start_time = datetime.combine( + t.cast( + # This will be always a date for us, so casting. + date, + st.date_input( + "Start time", + value=datetime.now() - timedelta(weeks=settings.PAST_N_WEEKS), + ), + ), + datetime.min.time(), + ).replace(tzinfo=pytz.UTC) + market_type: MarketType = check_not_none( + st.selectbox(label="Market type", options=list(MarketType), index=0) + ) + + if market_type != MarketType.MANIFOLD: + st.warning("Only Manifold markets are supported for now.") + return + + st.subheader("Market resolution") + open_markets = get_manifold_markets_dated(oldest_date=start_time, filter_="open") + resolved_markets = [ + m + for m in get_manifold_markets_dated(oldest_date=start_time, filter_="resolved") + if m.resolution not in ("CANCEL", "MKT") + ] + monitor_market(open_markets=open_markets, resolved_markets=resolved_markets) + + with st.spinner("Loading Manifold agents..."): + agents: list[DeployedManifoldAgent] = [] + for key in settings.MANIFOLD_API_KEYS: + manifold_user = get_authenticated_user(key) + agents.append( + DeployedManifoldAgent( + name=manifold_user.name, + manifold_user_id=manifold_user.id, + start_time=start_time, + ) + ) + + st.subheader("Agent bets") + for agent in agents: + with st.expander(f"Agent: '{agent.name}'"): + monitor_agent(agent) diff --git a/pyproject.toml b/pyproject.toml index 8165dd1b..6237fa67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ tqdm = "^4.66.2" langchain-community = ">=0.0.19" scikit-learn = "^1.4.0" tabulate = "^0.9.0" +types-pytz = "^2024.1.0.20240203" [tool.poetry.group.dev.dependencies] pytest = "*"