chore(exchanges): working with okx data

exchanges/constants/urls.py

@@ -1,5 +1,6 @@
 BINANCE_API_OPTIONS_URL = "https://eapi.binance.com"
 BINANCE_API_FUTURES_URL = "https://fapi.binance.com/fapi/v1/premiumIndex"
+BINANCE_API_SPOT_URL = "https://api.binance.com"
 ByBit_API_URL = "https://api.bybit.com/v2/public/symbols"
 OKX_API_URL = "https://www.okex.com/api/spot/v3/instruments"
 KRAKEN_API_URL = "https://api.kraken.com/0/public/AssetPairs"

exchanges/exchange_manager.py

@@ -24,7 +24,7 @@ def __init__(self, exchange_id, pairs_to_load, market_types):
         self.futures_data = pd.DataFrame()
 
     def fetch_binance_symbols(self):
-        binance_option_symbols = self.binance_fetcher.fetch_symbols()
+        binance_option_symbols = self.binance_fetcher.fetch_options_symbols()
         return binance_option_symbols
 
     def load_specific_pairs(self) -> pd.DataFrame:

exchanges/fetchers/binance_fetcher.py

@@ -1,7 +1,10 @@
 import pandas as pd
 
-from exchanges.constants.urls import BINANCE_API_OPTIONS_URL, BINANCE_API_FUTURES_URL
-
+from exchanges.constants.urls import (
+    BINANCE_API_OPTIONS_URL,
+    BINANCE_API_FUTURES_URL,
+    BINANCE_API_SPOT_URL,
+)
 
 import logging
 import requests
@@ -31,21 +34,38 @@ def get_response(url):
 
     @staticmethod
     def fetch_options_symbols():
-        data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/exchangeInfo")["optionSymbols"]
+        data = BinanceFetcher.get_response(
+            BINANCE_API_OPTIONS_URL + "/eapi/v1/exchangeInfo"
+        )["optionSymbols"]
         data_df = pd.DataFrame(data)
         # all symbols with BTC-
         symbols = data_df["symbol"].loc[data_df["symbol"].str.contains("BTC-")]
-        return symbols
+        return symbols.tolist()
 
     @staticmethod
-    def fetch_mark_price():
-        data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/mark")
-        data_df = pd.DataFrame(data)
-        # get all where BTC- is in the symbol
-        return data_df.loc[data_df["symbol"].str.contains("BTC-")]
+    def fetch_mark_and_underlying_price():
+        mark_prices = BinanceFetcher.get_response(
+            BINANCE_API_OPTIONS_URL + "/eapi/v1/mark"
+        )
+        underlying_price = BinanceFetcher.get_response(
+            BINANCE_API_SPOT_URL + "/api/v3/ticker/price"
+        )
+        underlying_price_df = pd.DataFrame(underlying_price)
+        data_df = pd.DataFrame(mark_prices)
+        data_df = data_df.loc[data_df["symbol"].str.contains("BTC-")]
+
+        # Ensure that only the BTCUSDT price is fetched to match "BTC-" symbols
+        ud_price = underlying_price_df.loc[
+            underlying_price_df["symbol"] == "BTCUSDT", "price"
+        ].iloc[0]
 
+        data_df["underlying_price"] = float(ud_price)
+        data_df.rename(columns={"markPrice": "mark_price"}, inplace=True)
 
+        # Convert "mark_price" to float
+        data_df["mark_price"] = data_df["mark_price"].astype(float)
 
+        return data_df[["symbol", "mark_price", "underlying_price"]]
 
     @staticmethod
     def fetch_futures_symbols():
@@ -54,4 +74,4 @@ def fetch_futures_symbols():
             return [
                 res.get("symbol") for res in data if "BTCUSDT_" in res.get("symbol", "")
             ]
-        return []
+        return []

exchanges/fetchers/option_fetcher.py

@@ -4,10 +4,13 @@
 import pandas as pd
 import requests
 
+from exchanges.fetchers.binance_fetcher import BinanceFetcher
+
 
 class OptionFetcher:
     def __init__(self, exchange):
         self.exchange = exchange
+        self.binance_fetcher = BinanceFetcher()
 
     def fetch_market_data(
         self, market_symbols: list[str], exchange_name: str
@@ -23,9 +26,6 @@ def fetch_market_data(
         try:
             all_tickers = self.exchange.fetch_tickers(market_symbols)
             tickers_df = pd.DataFrame(all_tickers).transpose()
-            tickers_df.to_json(
-                f"{exchange_name}_raw_data_options.json", orient="records", indent=4
-            )
             if exchange_name == "Deribit":
                 return self.process_deribit_data(tickers_df)
             elif exchange_name == "OKX":
@@ -40,37 +40,31 @@ def fetch_market_data(
             return pd.DataFrame()
 
     def process_deribit_data(self, df: pd.DataFrame) -> pd.DataFrame:
-        # Convert 'info' column to separate columns for easier manipulation
         info_df = pd.json_normalize(df["info"])
         df = df.reset_index(drop=True)
 
-        # Replace 'None' with 0.0 and convert to float for 'bid' and 'ask'
         df["bid"] = pd.to_numeric(df["bid"], errors="coerce").fillna(0.0)
         df["ask"] = pd.to_numeric(df["ask"], errors="coerce").fillna(0.0)
 
-        # Convert 'mark_price' from info_df to numeric and update in df
         df["mark_price"] = pd.to_numeric(info_df["mark_price"], errors="coerce").fillna(
             0.0
         )
 
-        # Assuming info_df and df are aligned by index after pd.json_normalize
         underlying_prices = pd.to_numeric(
             info_df["underlying_price"], errors="coerce"
         ).fillna(0.0)
 
-        # Adjust 'bid' and 'ask' based on 'underlying_prices'
         df["bid"] *= underlying_prices
         df["ask"] *= underlying_prices
         df["mark_price"] *= underlying_prices
 
         df["underlying_price"] = underlying_prices
 
-        # Convert timestamp to datetime
         df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms")
 
-        # Efficient expiry date parsing (assuming this can be vectorized or is already efficient)
         df["expiry"] = df["symbol"].apply(self.date_parser)
-
+        df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S")
+        df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S")
         # Calculate YTM
         df["YTM"] = (df["expiry"] - df["datetime"]) / np.timedelta64(1, "Y")
 
@@ -84,6 +78,8 @@ def process_deribit_data(self, df: pd.DataFrame) -> pd.DataFrame:
                 "datetime",
                 "expiry",
                 "YTM",
+                "datetime_hum",
+                "expiry_hum",
                 "underlying_price",
             ]
         ]
@@ -106,16 +102,19 @@ def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame:
         df["ask"] *= df["underlying_price"]
         df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms")
         df["expiry"] = df["symbol"].apply(self.date_parser)
+        df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S")
+        df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S")
         df["YTM"] = (df["expiry"] - df["datetime"]) / np.timedelta64(1, "Y")
 
         # Merge the mark prices into the df based on the new 'symbol'
         df = df.merge(mark_prices_df[["symbol", "markPx"]], on="symbol", how="left")
 
         # Rename the 'markPx' column to 'mark_price' for clarity (optional)
         df.rename(columns={"markPx": "mark_price"}, inplace=True)
-        df["mark_price"] = pd.to_numeric(df["mark_price"], errors="coerce").fillna(0.0) * df[
-            "underlying_price"
-        ]
+        df["mark_price"] = (
+            pd.to_numeric(df["mark_price"], errors="coerce").fillna(0.0)
+            * df["underlying_price"]
+        )
 
         # Select and return the desired columns, including the new 'mark_price'
         return df[
@@ -127,20 +126,43 @@ def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame:
                 "underlying_price",
                 "datetime",
                 "expiry",
+                "datetime_hum",
+                "expiry_hum",
                 "YTM",
             ]
         ]
 
     def process_binance_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        df.to_json("binance_data.json", orient="records", indent=4)
         # Assuming 'info' contains the required details
-        print(df)
+        prices = self.binance_fetcher.fetch_mark_and_underlying_price()
+        prices["symbol"] = prices["symbol"].apply(self.transform_symbol_format)
+
         df["bid"] = df["info"].apply(lambda x: float(x.get("bidPrice", 0)))
         df["ask"] = df["info"].apply(lambda x: float(x.get("askPrice", 0)))
-        df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms")
+
+        df["datetime"] = pd.to_datetime(df["datetime"], unit="ms")
         df["expiry"] = df["symbol"].apply(self.date_parser)
+        df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S")
+        df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S")
         df["YTM"] = (df["expiry"] - df["timestamp"]) / np.timedelta64(1, "Y")
+        # Merge the prices into the df based on the 'symbol'
+        df = df.merge(prices, on="symbol", how="left")
 
-        return df[["symbol", "bid", "ask", "timestamp", "expiry", "YTM"]]
+        return df[
+            [
+                "symbol",
+                "bid",
+                "ask",
+                "mark_price",
+                "underlying_price",
+                "timestamp",
+                "expiry",
+                "datetime_hum",
+                "expiry_hum",
+                "YTM",
+            ]
+        ]
 
     @staticmethod
     def date_parser(symbol: str) -> pd.Timestamp:
@@ -168,3 +190,8 @@ def convert_inst_id_to_symbol(inst_id: str) -> str:
         # Reassemble into the symbol format
         symbol = f"{currency}:{parts[0]}-{date}-{strike_price}-{option_type}"
         return symbol
+
+    @staticmethod
+    def transform_symbol_format(symbol):
+        parts = symbol.split("-")
+        return f"{parts[0]}/USDT:USDT-{parts[1]}-{parts[2]}-{parts[3]}"

exchanges/handlers/merge.py

@@ -20,14 +20,8 @@ def handle(
         self, options_market: List[str], future_market: List[str] | None
     ) -> pd.DataFrame:
         options_data = self.option_market_handler.handle(options_market)
-        options_data.to_json(
-            f"{self.exchange}_options_data.json", orient="records", indent=4
-        )
         if future_market:
             futures_data = self.future_market_handler.handle(future_market)
-            futures_data.to_json(
-                f"{self.exchange}_futures_data.json", orient="records", indent=4
-            )
             merged_data = pd.concat([options_data, futures_data], ignore_index=True)
         else:
             merged_data = options_data
@@ -36,16 +30,12 @@ def handle(
         implied_interest_rates = self.processing.calculate_implied_interest_rates(
             valid_quotes
         )
+        implied_interest_rates.to_json(
+            f"{self.exchange}_implied_interest_rates.json", orient="records", indent=4
+        )
 
         spreads = self.processing.calculate_spreads(valid_quotes)
         remove_large_spreads = self.processing.remove_large_spreads(spreads)
         spreads.to_json(f"{self.exchange}_spreads.json", orient="records", indent=4)
-        remove_large_spreads.to_json(
-            f"{self.exchange}_remove_large_spreads.json", orient="records", indent=4
-        )
-
-        implied_interest_rates.to_json(
-            f"{self.exchange}_implied_interest_rates.json", orient="records", indent=4
-        )
 
         return merged_data

exchanges/main.py

@@ -17,12 +17,10 @@ def main():
         binance = BinanceManager(
             pairs_to_load=["BTC/USD:BTC"], market_types=["option", "future"]
         )
-        okx = OKXManager(
-            pairs_to_load=["BTC/USD:BTC"], market_types=["option"]
-        )
+        okx = OKXManager(pairs_to_load=["BTC/USD:BTC"], market_types=["option"])
         results = pd.DataFrame()
 
-        for manager in [binance]:
+        for manager in [binance, deribit, okx]:
             results = pd.concat(
                 [results, manager.load_specific_pairs()], ignore_index=True
             )