Add streamlit example of monitoring a manifold agent

examples/cloud_deployment/gcp/deploy.py

@@ -13,9 +13,9 @@
 if __name__ == "__main__":
     current_dir = os.path.dirname(os.path.realpath(__file__))
     fname = deploy_to_gcp(
-        requirements_file=f"{current_dir}/../../pyproject.toml",
+        requirements_file=f"{current_dir}/../../../pyproject.toml",
         extra_deps=[
-            "git+https://github.com/gnosis/prediction-market-agent.git@evan/deploy-agent"
+            "git+https://github.com/gnosis/prediction-market-agent-tooling.git"
         ],
         function_file=f"{current_dir}/agent.py",
         market_type=MarketType.MANIFOLD,
@@ -30,8 +30,8 @@
     response = run_deployed_gcp_function(fname)
     assert response.ok
 
-    # Schedule the function
-    schedule_deployed_gcp_function(fname, cron_schedule="* * * * *")
+    # Schedule the function to run once every 2 hours
+    schedule_deployed_gcp_function(fname, cron_schedule="0 */2 * * *")
 
     # Delete the function
     remove_deployed_gcp_function(fname)

examples/monitor/monitor.py

@@ -1,15 +1,17 @@
 from datetime import datetime, timedelta
+import pytz
+
 from prediction_market_agent_tooling.markets.manifold import get_authenticated_user
-from prediction_market_agent_tooling.monitor.monitor import (
+from prediction_market_agent_tooling.monitor.markets.manifold import (
     DeployedManifoldAgent,
-    monitor_agent,
-)
-
-
-agent = DeployedManifoldAgent(
-    name="foo",
-    start_time=datetime.now() - timedelta(weeks=2),
-    manifold_user=get_authenticated_user(),
 )
+from prediction_market_agent_tooling.monitor.monitor import monitor_agent
 
-monitor_agent(agent)
+if __name__ == "__main__":
+    start_time = datetime.now() - timedelta(weeks=1)
+    agent = DeployedManifoldAgent(
+        name="foo",
+        start_time=start_time.astimezone(pytz.UTC),
+        manifold_user_id=get_authenticated_user().id,
+    )
+    monitor_agent(agent)

prediction_market_agent_tooling/markets/data_models.py

@@ -1,8 +1,10 @@
+from datetime import datetime
 from decimal import Decimal
 from enum import Enum
 import typing as t
 from pydantic import BaseModel
 from web3 import Web3
+
 from prediction_market_agent_tooling.gtypes import (
     USD,
     HexAddress,
@@ -13,7 +15,6 @@
     xDai,
     Wei,
 )
-from datetime import datetime
 
 
 class Currency(str, Enum):
@@ -26,6 +27,28 @@ class BetAmount(BaseModel):
     currency: Currency
 
 
+class ProfitAmount(BaseModel):
+    amount: Decimal
+    currency: Currency
+
+
+class Bet(BaseModel):
+    amount: BetAmount
+    outcome: bool
+    created_time: datetime
+    market_question: str
+
+
+class ResolvedBet(Bet):
+    market_outcome: bool
+    resolved_time: datetime
+    profit: ProfitAmount
+
+    @property
+    def is_correct(self) -> bool:
+        return self.outcome == self.market_outcome
+
+
 class AgentMarket(BaseModel):
     """
     Common market class that can be created from vendor specific markets.
@@ -130,6 +153,8 @@ class ManifoldMarket(BaseModel):
     creatorName: str
     creatorUsername: str
     isResolved: bool
+    resolution: t.Optional[str] = None
+    resolutionTime: t.Optional[datetime] = None
     lastBetTime: datetime
     lastCommentTime: t.Optional[datetime] = None
     lastUpdatedTime: datetime
@@ -228,11 +253,28 @@ class ManifoldBet(BaseModel):
     loanAmount: Mana
     orderAmount: t.Optional[Mana] = None
     fills: t.Optional[list[ManifoldBetFills]] = None
-    createdTime: int
+    createdTime: datetime
     outcome: str
 
 
-class Bet(BaseModel):
-    amount: BetAmount
-    outcome: bool
-    created_time: datetime
+class ManifoldContractMetric(BaseModel):
+    """
+    https://docs.manifold.markets/api#get-v0marketmarketidpositions
+    """
+
+    contractId: str
+    hasNoShares: bool
+    hasShares: bool
+    hasYesShares: bool
+    invested: Decimal
+    loan: Decimal
+    maxSharesOutcome: t.Optional[str]
+    payout: Decimal
+    profit: Decimal
+    profitPercent: Decimal
+    totalShares: dict[str, Decimal]
+    userId: str
+    userUsername: str
+    userName: str
+    userAvatarUrl: str
+    lastBetTime: datetime

prediction_market_agent_tooling/markets/manifold.py

@@ -1,11 +1,17 @@
+from datetime import datetime
 import requests
 import typing as t
 from prediction_market_agent_tooling.gtypes import Mana
 from prediction_market_agent_tooling.config import APIKeys
 from prediction_market_agent_tooling.markets.data_models import (
+    ProfitAmount,
+    ResolvedBet,
+    BetAmount,
+    Currency,
     ManifoldBet,
     ManifoldMarket,
     ManifoldUser,
+    ManifoldContractMetric,
 )
 
 """
@@ -84,10 +90,55 @@ def get_authenticated_user() -> ManifoldUser:
     return ManifoldUser.model_validate(response.json())
 
 
-def get_bets(user_id: str) -> list[ManifoldBet]:
+def get_manifold_market(market_id: str) -> ManifoldMarket:
+    url = f"https://api.manifold.markets/v0/market/{market_id}"
+    response = requests.get(url)
+    response.raise_for_status()
+    return ManifoldMarket.model_validate(response.json())
+
+
+def get_resolved_manifold_bets(
+    user_id: str,
+    start_time: datetime,
+    end_time: t.Optional[datetime],
+) -> list[ManifoldBet]:
     url = "https://api.manifold.markets/v0/bets"
 
     params: dict[str, str] = {"userId": user_id}
     response = requests.get(url, params=params)
     response.raise_for_status()
-    return [ManifoldBet.model_validate(x) for x in response.json()]
+    bets = [ManifoldBet.model_validate(x) for x in response.json()]
+    bets = [b for b in bets if b.createdTime >= start_time]
+    if end_time:
+        bets = [b for b in bets if b.createdTime < end_time]
+    bets = [b for b in bets if get_manifold_market(b.contractId).isResolved]
+    return bets
+
+
+def manifold_to_generic_resolved_bet(bet: ManifoldBet) -> ResolvedBet:
+    market = get_manifold_market(bet.contractId)
+    if not market.isResolved:
+        raise ValueError(f"Market {market.id} is not resolved.")
+
+    # Get the profit for this bet from the corresponding position
+    positions = get_market_positions(market.id, bet.userId)
+    bet_position = next(p for p in positions if p.contractId == bet.contractId)
+    profit = bet_position.profit
+
+    return ResolvedBet(
+        amount=BetAmount(amount=bet.amount, currency=Currency.Mana),
+        outcome=bet.outcome == "YES",
+        created_time=bet.createdTime,
+        market_question=market.question,
+        market_outcome=market.resolution,
+        resolved_time=market.resolutionTime,
+        profit=ProfitAmount(amount=profit, currency=Currency.Mana),
+    )
+
+
+def get_market_positions(market_id: str, user_id: str) -> list[ManifoldContractMetric]:
+    url = f"https://api.manifold.markets/v0/market/{market_id}/positions"
+    params = {"userId": user_id}
+    response = requests.get(url, params=params)
+    response.raise_for_status()
+    return [ManifoldContractMetric.model_validate(x) for x in response.json()]

prediction_market_agent_tooling/monitor/markets/manifold.py

@@ -0,0 +1,18 @@
+from prediction_market_agent_tooling.markets.data_models import ResolvedBet
+from prediction_market_agent_tooling.markets.manifold import (
+    get_resolved_manifold_bets,
+    manifold_to_generic_resolved_bet,
+)
+from prediction_market_agent_tooling.monitor.monitor import DeployedAgent
+
+
+class DeployedManifoldAgent(DeployedAgent):
+    manifold_user_id: str
+
+    def get_resolved_bets(self) -> list[ResolvedBet]:
+        manifold_bets = get_resolved_manifold_bets(
+            user_id=self.manifold_user_id,
+            start_time=self.start_time,
+            end_time=None,
+        )
+        return [manifold_to_generic_resolved_bet(b) for b in manifold_bets]

prediction_market_agent_tooling/monitor/monitor.py

@@ -1,30 +1,68 @@
+import altair as alt
 from datetime import datetime
 from pydantic import BaseModel
+import pandas as pd
+import pytz
+import streamlit as st
 import typing as t
 
-from prediction_market_agent_tooling.markets.data_models import Bet, ManifoldUser
-from prediction_market_agent_tooling.markets.manifold import get_bets
+from prediction_market_agent_tooling.markets.data_models import ResolvedBet
 
 
 class DeployedAgent(BaseModel):
     name: str
-    start_time: datetime
+    start_time: datetime = datetime.now().astimezone(tz=pytz.UTC)
     end_time: t.Optional[datetime] = None
 
-    def get_bets(self) -> list[Bet]:
+    def get_resolved_bets(self) -> list[ResolvedBet]:
         raise NotImplementedError("Subclasses must implement this method.")
 
 
-class DeployedManifoldAgent(DeployedAgent):
-    manifold_user: ManifoldUser
+def monitor_agent(agent: DeployedAgent) -> None:
+    agent_bets = agent.get_resolved_bets()
+    bets_info = {
+        "Market Question": [bet.market_question for bet in agent_bets],
+        "Bet Amount": [bet.amount.amount for bet in agent_bets],
+        "Bet Outcome": [bet.outcome for bet in agent_bets],
+        "Created Time": [bet.created_time for bet in agent_bets],
+        "Resolved Time": [bet.resolved_time for bet in agent_bets],
+        "Is Correct": [bet.is_correct for bet in agent_bets],
+        "Profit": [bet.profit.amount for bet in agent_bets],
+    }
+    bets_df = pd.DataFrame(bets_info).sort_values(by="Resolved Time")
 
-    def get_bets(self) -> list[Bet]:
-        return get_bets(self.manifold_user.id)
+    st.set_page_config(layout="wide")
+    st.title(f"Monitoring Agent: '{agent.name}'")
 
+    # Metrics
+    col1, col2 = st.columns(2)
+    col1.metric(label="Number of bets", value=f"{len(agent_bets)}")
+    col2.metric(label="% Correct", value=f"{100 * bets_df['Is Correct'].mean():.2f}%")
 
-def monitor_agent(agent: DeployedAgent) -> None:
-    agent_bets = agent.get_bets()
-    print(f"Agent {agent.name} has {len(agent_bets)} bets.")
-    print(f"bet0: {agent_bets[0]}")
-    # TODO Get the bets from agent.starttime to agent.endtime (or now if endtime is None)
-    # TODO calculate the accuracy of last 10 bets for every day, and display it in a graph in streamlit app
+    # Chart of cumulative profit per day
+    profit_info = {
+        "Time": bets_df["Resolved Time"],
+        "Cumulative Profit": bets_df["Profit"].astype(float),
+    }
+    profit_df = pd.DataFrame(profit_info)
+    profit_df["Date"] = pd.to_datetime(profit_df["Time"].dt.date)
+    profit_df = (
+        profit_df.groupby("Date")["Cumulative Profit"].sum().cumsum().reset_index()
+    )
+    profit_df["Cumulative Profit"] = profit_df["Cumulative Profit"].astype(float)
+    st.empty()
+    st.altair_chart(
+        alt.Chart(profit_df)
+        .mark_line()
+        .encode(
+            x=alt.X("Date", axis=alt.Axis(format="%Y-%m-%d"), title=None),
+            y=alt.Y("Cumulative Profit", axis=alt.Axis(format=".2f")),
+        )
+        .interactive(),
+        use_container_width=True,
+    )
+
+    # Table of resolved bets
+    st.empty()
+    st.subheader("Resolved Bet History")
+    st.table(bets_df)

pyproject.toml

@@ -10,7 +10,7 @@ buy_omen = "scripts.bet_omen:buy"
 sell_omen = "scripts.bet_omen:sell"
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
+python = ">=3.10,<3.12"
 typer = "^0.9.0"
 types-requests = "^2.31.0.20240106"
 google-cloud-functions = "^1.16.0"
@@ -22,6 +22,7 @@ web3 = "^6.15.1"
 eth-typing = "^4.0.0"
 pydantic-settings = "^2.1.0"
 numpy = "^1.26.4"
+streamlit = "^1.31.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "*"