Merge pull request #2182 from Drakkar-Software/dev

Master merge

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)*
 
+## [0.4.34] - 2023-01-14
+### Added
+- Websockets: support for many more feeds and exchanges
+### Updated
+- Websockets: migrate form cryptofeed to ccxt pro
+- Web interface display speed
+- Coins logo display
+- Mobile display
+
 ## [0.4.33] - 2023-01-02
 ### Added
 - Profile selector

README.md

@@ -1,4 +1,4 @@
-# OctoBot [0.4.33](https://octobot.click/gh-changelog)
+# OctoBot [0.4.34](https://octobot.click/gh-changelog)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot.svg)](https://octobot.click/gh-pypi)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/e07fb190156d4efb8e7d07aaa5eff2e1)](https://app.codacy.com/gh/Drakkar-Software/OctoBot?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot&utm_campaign=Badge_Grade_Dashboard)[![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot)
 [![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg)](https://octobot.click/gh-dockerhub)

exchanges_tests/__init__.py

@@ -16,6 +16,7 @@
 import contextlib
 import os
 import dotenv
+import mock
 
 import trading_backend
 import octobot_commons.constants as commons_constants
@@ -24,15 +25,35 @@
 import octobot_commons.tests.test_config as test_config
 import octobot_trading.api as trading_api
 import octobot_trading.exchanges as exchanges
+import octobot_trading.constants as trading_constants
 import octobot_trading.enums as enums
 import octobot_trading.errors as errors
+import octobot_trading.exchange_channel as exchange_channel
+import octobot_trading.personal_data as personal_data
 import octobot_tentacles_manager.constants as tentacles_manager_constants
 import tests.test_utils.config as test_utils_config
 
 
 LOADED_EXCHANGE_CREDS_ENV_VARIABLES = False
 
 
+class ExchangeChannelMock:
+    def __init__(self, exchange_manager, name):
+        self.exchange_manager = exchange_manager
+        self.name = name
+
+        self.get_internal_producer = mock.Mock(
+            return_value=mock.Mock(
+                update_order_from_exchange=mock.AsyncMock(),
+                send=mock.AsyncMock(),
+            )
+        )
+        self.get_consumers = mock.Mock(return_value=[mock.Mock()])
+
+    def get_name(self):
+        return self.name
+
+
 @contextlib.asynccontextmanager
 async def get_authenticated_exchange_manager(exchange_name, exchange_tentacle_name, config=None):
     _load_exchange_creds_env_variables_if_necessary()
@@ -60,6 +81,7 @@ async def get_authenticated_exchange_manager(exchange_name, exchange_tentacle_na
     exchange_manager_instance.exchange_backend = trading_backend.exchange_factory.create_exchange_backend(
         exchange_manager_instance.exchange
     )
+    set_mocked_required_channels(exchange_manager_instance)
     try:
         yield exchange_manager_instance
     except errors.UnreachableExchange as err:
@@ -73,6 +95,13 @@ async def get_authenticated_exchange_manager(exchange_name, exchange_tentacle_na
         await asyncio_tools.wait_asyncio_next_cycle()
 
 
+def set_mocked_required_channels(exchange_manager):
+    # disable waiting time as order refresh is mocked
+    personal_data.State.PENDING_REFRESH_INTERVAL = 0
+    for channel in (trading_constants.ORDERS_CHANNEL, trading_constants.BALANCE_CHANNEL):
+        exchange_channel.set_chan(ExchangeChannelMock(exchange_manager, channel), channel)
+
+
 def get_tentacles_setup_config_with_exchange(exchange_tentacle_name):
     setup_config = test_utils_config.load_test_tentacles_config()
     setup_config.tentacles_activation[tentacles_manager_constants.TENTACLES_TRADING_PATH][exchange_tentacle_name] = True

exchanges_tests/abstract_authenticated_exchange_tester.py

@@ -46,6 +46,7 @@ class AbstractAuthenticatedExchangeTester:
     NO_FEE_ON_GET_CLOSED_ORDERS = False
     OPEN_ORDERS_IN_CLOSED_ORDERS = False
     MARKET_FILL_TIMEOUT = 15
+    OPEN_TIMEOUT = 15
     CANCEL_TIMEOUT = 15
     EDIT_TIMEOUT = 15
 
@@ -71,8 +72,7 @@ async def inner_test_create_and_cancel_limit_orders(self):
         buy_limit = await self.create_limit_order(price, size, trading_enums.TradeOrderSide.BUY)
         self.check_created_limit_order(buy_limit, price, size, trading_enums.TradeOrderSide.BUY)
         assert await self.order_in_open_orders(open_orders, buy_limit)
-        if await self.cancel_order(buy_limit) is trading_enums.OrderStatus.PENDING_CANCEL:
-            await self.wait_for_cancel(buy_limit)
+        await self.cancel_order(buy_limit)
         assert await self.order_not_in_open_orders(open_orders, buy_limit)
 
     async def test_create_and_fill_market_orders(self):
@@ -113,8 +113,7 @@ async def inner_test_create_and_cancel_stop_orders(self):
         stop_loss_from_get_order = await self.get_order(stop_loss.order_id)
         self.check_created_stop_order(stop_loss_from_get_order, price, size, trading_enums.TradeOrderSide.SELL)
         assert await self.order_in_open_orders(open_orders, stop_loss)
-        if await self.cancel_order(stop_loss) is trading_enums.OrderStatus.PENDING_CANCEL:
-            await self.wait_for_cancel(stop_loss)
+        await self.cancel_order(stop_loss)
         assert await self.order_not_in_open_orders(open_orders, stop_loss)
 
     async def test_get_my_recent_trades(self):
@@ -157,8 +156,7 @@ async def inner_test_edit_limit_order(self):
         await self.wait_for_edit(sell_limit, edited_size)
         sell_limit = await self.get_order(sell_limit.order_id)
         self.check_created_limit_order(sell_limit, edited_price, edited_size, trading_enums.TradeOrderSide.SELL)
-        if await self.cancel_order(sell_limit) is trading_enums.OrderStatus.PENDING_CANCEL:
-            await self.wait_for_cancel(sell_limit)
+        await self.cancel_order(sell_limit)
         assert await self.order_not_in_open_orders(open_orders, sell_limit)
 
     async def test_edit_stop_order(self):
@@ -184,8 +182,7 @@ async def inner_test_edit_stop_order(self):
         await self.wait_for_edit(stop_loss, edited_size)
         stop_loss = await self.get_order(stop_loss.order_id)
         self.check_created_stop_order(stop_loss, edited_price, edited_size, trading_enums.TradeOrderSide.SELL)
-        if await self.cancel_order(stop_loss) is trading_enums.OrderStatus.PENDING_CANCEL:
-            await self.wait_for_cancel(stop_loss)
+        await self.cancel_order(stop_loss)
         assert await self.order_not_in_open_orders(open_orders, stop_loss)
 
     async def test_create_bundled_orders(self):
@@ -212,16 +209,15 @@ async def inner_test_create_bundled_orders(self):
         params.update(
             await self.exchange_manager.trader.bundle_chained_order_with_uncreated_order(market_order, take_profit)
         )
-        buy_market = await self.exchange_manager.trader.create_order(market_order, params=params)
+        buy_market = await self._create_order_on_exchange(market_order, params=params)
         self.check_created_market_order(buy_market, size, trading_enums.TradeOrderSide.BUY)
         await self.wait_for_fill(buy_market)
         created_orders = [stop_loss, take_profit]
         fetched_conditional_orders = await self.get_similar_orders_in_open_orders(open_orders, created_orders)
         for fetched_conditional_order in fetched_conditional_orders:
             # ensure stop loss / take profit is fetched in open orders
             # ensure stop loss / take profit cancel is working
-            if await self.cancel_order(fetched_conditional_order) is trading_enums.OrderStatus.PENDING_CANCEL:
-                await self.wait_for_cancel(fetched_conditional_order)
+            await self.cancel_order(fetched_conditional_order)
         for fetched_conditional_order in fetched_conditional_orders:
             assert await self.order_not_in_open_orders(open_orders, fetched_conditional_order)
         # close position
@@ -244,15 +240,15 @@ def check_duplicate(self, orders_or_trades):
             f"{o[trading_enums.ExchangeConstantsOrderColumns.ID.value]}"
             f"{o[trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value]}"
             f"{o[trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value]}"
+            f"{o[trading_enums.ExchangeConstantsOrderColumns.PRICE.value]}"
             for o in orders_or_trades
         }) == len(orders_or_trades)
 
     def check_raw_closed_orders(self, closed_orders):
         self.check_duplicate(closed_orders)
         for closed_order in closed_orders:
-            clean_order = self.exchange_manager.exchange.clean_order(closed_order)
             self.check_parsed_closed_order(
-                personal_data.create_order_instance_from_raw(self.exchange_manager.trader, clean_order)
+                personal_data.create_order_instance_from_raw(self.exchange_manager.trader, closed_order)
             )
 
     def check_parsed_closed_order(self, order: personal_data.Order):
@@ -278,9 +274,8 @@ def check_parsed_closed_order(self, order: personal_data.Order):
     def check_raw_trades(self, trades):
         self.check_duplicate(trades)
         for trade in trades:
-            clean_trade = self.exchange_manager.exchange.clean_trade(trade)
             self.check_parsed_trade(
-                personal_data.create_trade_instance_from_raw(self.exchange_manager.trader, clean_trade)
+                personal_data.create_trade_instance_from_raw(self.exchange_manager.trader, trade)
             )
 
     def check_parsed_trade(self, trade: personal_data.Trade):
@@ -294,7 +289,8 @@ def check_parsed_trade(self, trade: personal_data.Trade):
         if trade.status is not trading_enums.OrderStatus.CANCELED:
             assert trade.executed_quantity
             self.check_theoretical_cost(
-                symbols.parse_symbol(trade.symbol), trade.executed_quantity, trade.executed_price, trade.total_cost
+                symbols.parse_symbol(trade.symbol), trade.executed_quantity,
+                trade.executed_price, trade.total_cost
             )
 
     def check_theoretical_cost(self, symbol, quantity, price, cost):
@@ -393,11 +389,18 @@ async def create_order(self, price, current_price, size, side, order_type,
             side=side,
         )
         if push_on_exchange:
-            current_order = await self.exchange_manager.trader.create_order(current_order)
+            current_order = await self._create_order_on_exchange(current_order)
         if current_order is None:
             raise AssertionError("Error when creating order")
         return current_order
 
+    async def _create_order_on_exchange(self, order, params=None):
+        created_order = await self.exchange_manager.trader.create_order(order, params=params, wait_for_creation=False)
+        if created_order.status is trading_enums.OrderStatus.PENDING_CREATION:
+            await self.wait_for_open(created_order)
+            return await self.get_order(created_order.order_id)
+        return created_order
+
     def get_order_size(self, portfolio, price, symbol=None, order_size=None):
         order_size = order_size or self.ORDER_SIZE
         currency_quantity = portfolio[self.SETTLEMENT_CURRENCY][self.PORTFOLIO_TYPE_FOR_SIZE] \
@@ -479,8 +482,17 @@ def parse_is_filled(raw_order):
                                                                    trading_enums.OrderStatus.CLOSED}
         await self._get_order_until(order, parse_is_filled, self.MARKET_FILL_TIMEOUT)
 
+    def parse_order_is_not_pending(self, raw_order):
+        return personal_data.parse_order_status(raw_order) not in (trading_enums.OrderStatus.UNKNOWN, None)
+
+    async def wait_for_open(self, order):
+        await self._get_order_until(order, self.parse_order_is_not_pending, self.OPEN_TIMEOUT)
+
     async def wait_for_cancel(self, order):
-        await self._get_order_until(order, personal_data.parse_is_cancelled, self.CANCEL_TIMEOUT)
+        return personal_data.create_order_instance_from_raw(
+            self.exchange_manager.trader,
+            await self._get_order_until(order, personal_data.parse_is_cancelled, self.CANCEL_TIMEOUT)
+        )
 
     async def wait_for_edit(self, order, edited_quantity):
         def is_edited(row_order):
@@ -493,7 +505,7 @@ async def _get_order_until(self, order, validation_func, timeout):
         while time.time() - t0 < timeout:
             raw_order = await self.exchange_manager.exchange.get_order(order.order_id, order.symbol)
             if raw_order and validation_func(raw_order):
-                return
+                return raw_order
         raise TimeoutError(f"Order not filled within {timeout}s: {order}")
 
     async def order_in_open_orders(self, previous_open_orders, order):
@@ -532,7 +544,15 @@ async def order_not_in_open_orders(self, previous_open_orders, order):
         return True
 
     async def cancel_order(self, order):
-        return await self.exchange_manager.exchange.cancel_order(order.order_id, order.symbol)
+        cancelled_order = order
+        if not await self.exchange_manager.trader.cancel_order(order, wait_for_cancelling=False):
+            raise AssertionError("cancel_order returned False")
+        if order.status is trading_enums.OrderStatus.PENDING_CANCEL:
+            cancelled_order = await self.wait_for_cancel(order)
+        assert cancelled_order.status is trading_enums.OrderStatus.CANCELED
+        if cancelled_order.state is not None:
+            assert cancelled_order.is_cancelled()
+        return order
 
     def get_config(self):
         return {

octobot/__init__.py

@@ -16,5 +16,5 @@
 
 PROJECT_NAME = "OctoBot"
 AUTHOR = "Drakkar-Software"
-VERSION = "0.4.33"  # major.minor.revision
+VERSION = "0.4.34"  # major.minor.revision
 LONG_VERSION = f"{VERSION}"

octobot/community/feeds/community_mqtt_feed.py

@@ -64,7 +64,7 @@ def __init__(self, feed_url, authenticator):
         self._reconnect_task = None
         self._connect_task = None
         self._connected_at_least_once = False
-        self._processed_messages = set()
+        self._processed_messages = []
 
     async def start(self):
         self.should_stop = False
@@ -174,7 +174,7 @@ def _should_process(self, parsed_message):
             self.logger.debug(f"Ignored already processed message with id: "
                               f"{parsed_message[commons_enums.CommunityFeedAttrs.ID.value]}")
             return False
-        self._processed_messages.add(parsed_message[commons_enums.CommunityFeedAttrs.ID.value])
+        self._processed_messages.append(parsed_message[commons_enums.CommunityFeedAttrs.ID.value])
         if len(self._processed_messages) > self.MAX_MESSAGE_ID_CACHE_SIZE:
             self._processed_messages = [
                 message_id

requirements.txt

@@ -3,8 +3,8 @@ cython==0.29.32
 
 # Drakkar-Software requirements
 OctoBot-Commons==1.8.2
-OctoBot-Trading==2.3.4
-OctoBot-Evaluators==1.8.0
+OctoBot-Trading==2.3.7
+OctoBot-Evaluators==1.8.1
 OctoBot-Tentacles-Manager==2.8.1
 OctoBot-Services==1.4.1
 OctoBot-Backtesting==1.8.0

tests/unit_tests/community/test_authentication.py

@@ -424,11 +424,15 @@ def _auth_handler_mock_context_manager(*args):
 
 
 def test_check_auth(auth):
-    resp_mock = mock.Mock()
-    with mock.patch.object(requests, "post", mock.Mock(return_value=resp_mock)), \
+    mocked_resp = MockedResponse(json=EMAIL_RETURN, headers={auth.SESSION_HEADER: "hi"})
+
+    @contextlib.contextmanager
+    def get_mock(*_, **__):
+        yield mocked_resp
+    with mock.patch.object(auth._session, "get", get_mock), \
          mock.patch.object(auth, "_handle_auth_result", mock.Mock()) as handle_result_mock:
         auth._check_auth()
-        assert handle_result_mock.called_once_with(resp_mock.status_code, resp_mock.json(), resp_mock.headers)
+        handle_result_mock.assert_called_once_with(mocked_resp.status_code, mocked_resp.json(), mocked_resp.headers)
 
 
 @pytest.mark.asyncio
@@ -445,7 +449,7 @@ async def async_get(*_, **__):
     auth._aiohttp_session.get = async_get
     with mock.patch.object(auth, "_handle_auth_result", mock.Mock()) as handle_result_mock:
         await auth._async_check_auth()
-        assert handle_result_mock.called_once_with(resp_mock.status_code, "plop", resp_mock.headers)
+        handle_result_mock.assert_called_once_with(resp_mock.status, "plop", resp_mock.headers)
 
 
 def test_handle_auth_result(auth):