diff --git a/exchanges/consolidate_data.py b/exchanges/consolidate_data.py deleted file mode 100644 index cf2391c..0000000 --- a/exchanges/consolidate_data.py +++ /dev/null @@ -1,98 +0,0 @@ -import ccxt - -from exchanges.constants.utils import SPREAD_MULTIPLIER, SPREAD_MIN -from utils import handle_error - - -class ConsolidateData: - def __init__(self, exchange): - # Initialize with an exchange object and create a DataFilter instance. - self.exchange = exchange - self.data_fetcher = None # Initialize a placeholder for the DataFetcher. - - def standardize_data(self, symbol, order_book_data): - try: - from data_fetcher import DataFetcher - - self.data_fetcher = DataFetcher( - self.exchange - ) # Initialize the DataFetcher for the exchange. - - # Fetch the spot price and mark price for the given symbol. - spot_price, mark_price = self._fetch_prices(symbol) - - # Return empty if prices are not available. - if not spot_price or not mark_price: - return {} - - # Extract bids and asks from the order book data. - bids, asks = order_book_data.get("bids", []), order_book_data.get( - "asks", [] - ) - if not bids or not asks: - return {} - - # Determine the maximum bid price and minimum ask price. - max_bid, min_ask = self._get_max_min_prices(bids, asks) - - # Find the price of the selected option based on minimum ask spread. - selected_option_price = min(asks, key=lambda x: float(x[0]) - max_bid)[0] - mid_price = (max_bid + min_ask) / 2 # Calculate the mid-price. - - time_to_maturity = self.data_filter.calculate_time_to_maturity( - order_book_data - ) - - # Prepare the standardized data structure. - standardized_data = { - "symbol": symbol, - "order_book": order_book_data, - "current_spot_price": spot_price, - "mark_price": float(selected_option_price), - "mid_price": mid_price, - "time_to_maturity_years": time_to_maturity, - } - - # Check if the quote is valid and return the data, else return empty. - # return standardized_data if self._is_valid_quote(standardized_data) else {} - return standardized_data - - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - # Handle any network or exchange-related errors. - handle_error(f"Error standardizing data for symbol '{symbol}'", e) - return {} - - def _fetch_prices(self, symbol): - # Fetch and return the spot price and the mark price for the symbol. - spot_price = self.data_fetcher.fetch_price(symbol, "last") - mark_price = self.data_fetcher.fetch_mark_price(symbol) - return spot_price, mark_price - - def _get_max_min_prices(self, bids, asks): - # Calculate the maximum bid and minimum ask prices from order book data. - max_bid = max(float(bid[0]) for bid in bids) - min_ask = min(float(ask[0]) for ask in asks) - return max_bid, min_ask - - def _is_valid_quote(self, data): - # Validate the quote based on bid, ask, and mark prices and their spreads. - bid_price, ask_price = ( - data["order_book"]["bids"][0][0], - data["order_book"]["asks"][0][0], - ) - mark_price = data["mark_price"] - - # Calculate bid and ask spreads. - bid_spread, ask_spread = max(0, mark_price - bid_price), max( - 0, ask_price - mark_price - ) - mas = min(bid_spread, ask_spread) * SPREAD_MULTIPLIER # Maximum Allowed Spread. - gms = SPREAD_MIN * SPREAD_MULTIPLIER # Global Maximum Spread. - spread = bid_spread + ask_spread - - # Check if the spread is within limits and the price ordering is correct. - return ( - spread <= max(mas, gms) - and bid_price <= mark_price <= ask_price - and mark_price > 0 - ) diff --git a/exchanges/constants/urls.py b/exchanges/constants/urls.py index 1375fb4..859f756 100644 --- a/exchanges/constants/urls.py +++ b/exchanges/constants/urls.py @@ -1,5 +1,6 @@ BINANCE_API_OPTIONS_URL = "https://eapi.binance.com" BINANCE_API_FUTURES_URL = "https://fapi.binance.com/fapi/v1/premiumIndex" +BINANCE_API_SPOT_URL = "https://api.binance.com" ByBit_API_URL = "https://api.bybit.com/v2/public/symbols" OKX_API_URL = "https://www.okex.com/api/spot/v3/instruments" KRAKEN_API_URL = "https://api.kraken.com/0/public/AssetPairs" diff --git a/exchanges/data_filter.py b/exchanges/data_filter.py deleted file mode 100644 index e69de29..0000000 diff --git a/exchanges/exchange_manager.py b/exchanges/exchange_manager.py index 29eaf82..fb74dfc 100644 --- a/exchanges/exchange_manager.py +++ b/exchanges/exchange_manager.py @@ -24,7 +24,7 @@ def __init__(self, exchange_id, pairs_to_load, market_types): self.futures_data = pd.DataFrame() def fetch_binance_symbols(self): - binance_option_symbols = self.binance_fetcher.fetch_symbols() + binance_option_symbols = self.binance_fetcher.fetch_options_symbols() return binance_option_symbols def load_specific_pairs(self) -> pd.DataFrame: diff --git a/exchanges/fetchers/binance_fetcher.py b/exchanges/fetchers/binance_fetcher.py index b8366a3..90a3b1d 100644 --- a/exchanges/fetchers/binance_fetcher.py +++ b/exchanges/fetchers/binance_fetcher.py @@ -1,7 +1,10 @@ import pandas as pd -from exchanges.constants.urls import BINANCE_API_OPTIONS_URL, BINANCE_API_FUTURES_URL - +from exchanges.constants.urls import ( + BINANCE_API_OPTIONS_URL, + BINANCE_API_FUTURES_URL, + BINANCE_API_SPOT_URL, +) import logging import requests @@ -31,21 +34,38 @@ def get_response(url): @staticmethod def fetch_options_symbols(): - data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/exchangeInfo")["optionSymbols"] + data = BinanceFetcher.get_response( + BINANCE_API_OPTIONS_URL + "/eapi/v1/exchangeInfo" + )["optionSymbols"] data_df = pd.DataFrame(data) # all symbols with BTC- symbols = data_df["symbol"].loc[data_df["symbol"].str.contains("BTC-")] - return symbols + return symbols.tolist() @staticmethod - def fetch_mark_price(): - data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/mark") - data_df = pd.DataFrame(data) - # get all where BTC- is in the symbol - return data_df.loc[data_df["symbol"].str.contains("BTC-")] + def fetch_mark_and_underlying_price(): + mark_prices = BinanceFetcher.get_response( + BINANCE_API_OPTIONS_URL + "/eapi/v1/mark" + ) + underlying_price = BinanceFetcher.get_response( + BINANCE_API_SPOT_URL + "/api/v3/ticker/price" + ) + underlying_price_df = pd.DataFrame(underlying_price) + data_df = pd.DataFrame(mark_prices) + data_df = data_df.loc[data_df["symbol"].str.contains("BTC-")] + + # Ensure that only the BTCUSDT price is fetched to match "BTC-" symbols + ud_price = underlying_price_df.loc[ + underlying_price_df["symbol"] == "BTCUSDT", "price" + ].iloc[0] + data_df["underlying_price"] = float(ud_price) + data_df.rename(columns={"markPrice": "mark_price"}, inplace=True) + # Convert "mark_price" to float + data_df["mark_price"] = data_df["mark_price"].astype(float) + return data_df[["symbol", "mark_price", "underlying_price"]] @staticmethod def fetch_futures_symbols(): @@ -54,4 +74,4 @@ def fetch_futures_symbols(): return [ res.get("symbol") for res in data if "BTCUSDT_" in res.get("symbol", "") ] - return [] \ No newline at end of file + return [] diff --git a/exchanges/fetchers/option_fetcher.py b/exchanges/fetchers/option_fetcher.py index 1e404d1..ef0a4c2 100644 --- a/exchanges/fetchers/option_fetcher.py +++ b/exchanges/fetchers/option_fetcher.py @@ -4,10 +4,13 @@ import pandas as pd import requests +from exchanges.fetchers.binance_fetcher import BinanceFetcher + class OptionFetcher: def __init__(self, exchange): self.exchange = exchange + self.binance_fetcher = BinanceFetcher() def fetch_market_data( self, market_symbols: list[str], exchange_name: str @@ -23,9 +26,6 @@ def fetch_market_data( try: all_tickers = self.exchange.fetch_tickers(market_symbols) tickers_df = pd.DataFrame(all_tickers).transpose() - tickers_df.to_json( - f"{exchange_name}_raw_data_options.json", orient="records", indent=4 - ) if exchange_name == "Deribit": return self.process_deribit_data(tickers_df) elif exchange_name == "OKX": @@ -40,37 +40,31 @@ def fetch_market_data( return pd.DataFrame() def process_deribit_data(self, df: pd.DataFrame) -> pd.DataFrame: - # Convert 'info' column to separate columns for easier manipulation info_df = pd.json_normalize(df["info"]) df = df.reset_index(drop=True) - # Replace 'None' with 0.0 and convert to float for 'bid' and 'ask' df["bid"] = pd.to_numeric(df["bid"], errors="coerce").fillna(0.0) df["ask"] = pd.to_numeric(df["ask"], errors="coerce").fillna(0.0) - # Convert 'mark_price' from info_df to numeric and update in df df["mark_price"] = pd.to_numeric(info_df["mark_price"], errors="coerce").fillna( 0.0 ) - # Assuming info_df and df are aligned by index after pd.json_normalize underlying_prices = pd.to_numeric( info_df["underlying_price"], errors="coerce" ).fillna(0.0) - # Adjust 'bid' and 'ask' based on 'underlying_prices' df["bid"] *= underlying_prices df["ask"] *= underlying_prices df["mark_price"] *= underlying_prices df["underlying_price"] = underlying_prices - # Convert timestamp to datetime df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms") - # Efficient expiry date parsing (assuming this can be vectorized or is already efficient) df["expiry"] = df["symbol"].apply(self.date_parser) - + df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S") + df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S") # Calculate YTM df["YTM"] = (df["expiry"] - df["datetime"]) / np.timedelta64(1, "Y") @@ -84,6 +78,8 @@ def process_deribit_data(self, df: pd.DataFrame) -> pd.DataFrame: "datetime", "expiry", "YTM", + "datetime_hum", + "expiry_hum", "underlying_price", ] ] @@ -106,6 +102,8 @@ def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame: df["ask"] *= df["underlying_price"] df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms") df["expiry"] = df["symbol"].apply(self.date_parser) + df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S") + df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S") df["YTM"] = (df["expiry"] - df["datetime"]) / np.timedelta64(1, "Y") # Merge the mark prices into the df based on the new 'symbol' @@ -113,9 +111,10 @@ def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame: # Rename the 'markPx' column to 'mark_price' for clarity (optional) df.rename(columns={"markPx": "mark_price"}, inplace=True) - df["mark_price"] = pd.to_numeric(df["mark_price"], errors="coerce").fillna(0.0) * df[ - "underlying_price" - ] + df["mark_price"] = ( + pd.to_numeric(df["mark_price"], errors="coerce").fillna(0.0) + * df["underlying_price"] + ) # Select and return the desired columns, including the new 'mark_price' return df[ @@ -127,20 +126,43 @@ def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame: "underlying_price", "datetime", "expiry", + "datetime_hum", + "expiry_hum", "YTM", ] ] def process_binance_data(self, df: pd.DataFrame) -> pd.DataFrame: + df.to_json("binance_data.json", orient="records", indent=4) # Assuming 'info' contains the required details - print(df) + prices = self.binance_fetcher.fetch_mark_and_underlying_price() + prices["symbol"] = prices["symbol"].apply(self.transform_symbol_format) + df["bid"] = df["info"].apply(lambda x: float(x.get("bidPrice", 0))) df["ask"] = df["info"].apply(lambda x: float(x.get("askPrice", 0))) - df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") + + df["datetime"] = pd.to_datetime(df["datetime"], unit="ms") df["expiry"] = df["symbol"].apply(self.date_parser) + df["datetime_hum"] = df["datetime"].dt.strftime("%Y-%m-%d %H:%M:%S") + df["expiry_hum"] = df["expiry"].dt.strftime("%Y-%m-%d %H:%M:%S") df["YTM"] = (df["expiry"] - df["timestamp"]) / np.timedelta64(1, "Y") + # Merge the prices into the df based on the 'symbol' + df = df.merge(prices, on="symbol", how="left") - return df[["symbol", "bid", "ask", "timestamp", "expiry", "YTM"]] + return df[ + [ + "symbol", + "bid", + "ask", + "mark_price", + "underlying_price", + "timestamp", + "expiry", + "datetime_hum", + "expiry_hum", + "YTM", + ] + ] @staticmethod def date_parser(symbol: str) -> pd.Timestamp: @@ -168,3 +190,8 @@ def convert_inst_id_to_symbol(inst_id: str) -> str: # Reassemble into the symbol format symbol = f"{currency}:{parts[0]}-{date}-{strike_price}-{option_type}" return symbol + + @staticmethod + def transform_symbol_format(symbol): + parts = symbol.split("-") + return f"{parts[0]}/USDT:USDT-{parts[1]}-{parts[2]}-{parts[3]}" diff --git a/exchanges/handlers/merge.py b/exchanges/handlers/merge.py index 41d0f56..cf2f3e8 100644 --- a/exchanges/handlers/merge.py +++ b/exchanges/handlers/merge.py @@ -20,14 +20,8 @@ def handle( self, options_market: List[str], future_market: List[str] | None ) -> pd.DataFrame: options_data = self.option_market_handler.handle(options_market) - options_data.to_json( - f"{self.exchange}_options_data.json", orient="records", indent=4 - ) if future_market: futures_data = self.future_market_handler.handle(future_market) - futures_data.to_json( - f"{self.exchange}_futures_data.json", orient="records", indent=4 - ) merged_data = pd.concat([options_data, futures_data], ignore_index=True) else: merged_data = options_data @@ -36,16 +30,12 @@ def handle( implied_interest_rates = self.processing.calculate_implied_interest_rates( valid_quotes ) + implied_interest_rates.to_json( + f"{self.exchange}_implied_interest_rates.json", orient="records", indent=4 + ) spreads = self.processing.calculate_spreads(valid_quotes) remove_large_spreads = self.processing.remove_large_spreads(spreads) spreads.to_json(f"{self.exchange}_spreads.json", orient="records", indent=4) - remove_large_spreads.to_json( - f"{self.exchange}_remove_large_spreads.json", orient="records", indent=4 - ) - - implied_interest_rates.to_json( - f"{self.exchange}_implied_interest_rates.json", orient="records", indent=4 - ) return merged_data diff --git a/exchanges/main.py b/exchanges/main.py index ddbee2d..4de3483 100644 --- a/exchanges/main.py +++ b/exchanges/main.py @@ -17,12 +17,10 @@ def main(): binance = BinanceManager( pairs_to_load=["BTC/USD:BTC"], market_types=["option", "future"] ) - okx = OKXManager( - pairs_to_load=["BTC/USD:BTC"], market_types=["option"] - ) + okx = OKXManager(pairs_to_load=["BTC/USD:BTC"], market_types=["option"]) results = pd.DataFrame() - for manager in [binance]: + for manager in [binance, deribit, okx]: results = pd.concat( [results, manager.load_specific_pairs()], ignore_index=True )