diff --git a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py index f70507f84a..119ab2c1d4 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py @@ -142,6 +142,7 @@ class AbstractAuthenticatedExchangeTester: DEFAULT_MAX_DEFAULT_ORDERS_COUNT = trading_constants.DEFAULT_MAX_DEFAULT_ORDERS_COUNT DEFAULT_MAX_STOP_ORDERS_COUNT = trading_constants.DEFAULT_MAX_STOP_ORDERS_COUNT SLEEP_SECONDS_BEFORE_CHECKING_PORTFOLIO = 0 # used to wait before fetching portfolio after creating/cancelling an order + ENABLE_MARKET_ORDER_CONVERSION_CHECKS = False # set True to run after_market_order_created during market order tests # Implement all "test_[name]" methods, call super() to run the test, pass to ignore it. # Override the "inner_test_[name]" method to override a test content. @@ -714,6 +715,8 @@ async def inner_test_create_and_fill_market_orders(self): post_buy_portfolio = {} try: self.check_created_market_order(first_market_order, size, side) + if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS: + await self.after_market_order_created(first_market_order) filled_order = await self.wait_for_fill(first_market_order) parsed_filled_order = personal_data.create_order_instance_from_raw( self.exchange_manager.trader, @@ -735,6 +738,8 @@ async def inner_test_create_and_fill_market_orders(self): other_side = trading_enums.TradeOrderSide.SELL if side == trading_enums.TradeOrderSide.BUY else trading_enums.TradeOrderSide.BUY second_market_order = await self.create_market_order(current_price, mirror_size, other_side) self.check_created_market_order(second_market_order, mirror_size, other_side) + if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS: + await self.after_market_order_created(second_market_order) await self.wait_for_fill(second_market_order) await self.sleep_before_checking_portfolio() post_sell_portfolio = await self.get_portfolio() @@ -1702,6 +1707,9 @@ async def _get_recent_trades_until(self, validation_func, timeout): f"message: {message}" ) + async def after_market_order_created(self, market_order) -> None: + pass + async def wait_for_fill(self, order): def parse_is_filled(raw_order): return personal_data.parse_order_status(raw_order) in {trading_enums.OrderStatus.FILLED, diff --git a/octobot/cli.py b/octobot/cli.py index e4d7b59f4d..241d4602e1 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -57,6 +57,7 @@ import octobot.logger as octobot_logger import octobot.community as octobot_community import octobot.community.errors + import octobot.community.node_journal.lifecycle as journal_startup import octobot.limits as limits except ImportError as err: traceback.print_exc() @@ -478,8 +479,36 @@ def _log_startup_distribution_mode(logger, args) -> None: logger.info(startup_message) +def _record_cli_startup_failure( + error: Exception, + startup_phase: str, + config: configuration.Configuration | None = None, +) -> None: + try: + if config is not None: + distribution = configuration_manager.get_distribution(config.config) + elif constants.FORCED_DISTRIBUTION: + distribution = enums.OctoBotDistribution(constants.FORCED_DISTRIBUTION) + else: + return + if distribution is not enums.OctoBotDistribution.NODE: + return + journal_startup.record_node_startup_failed( + error, + startup_phase=startup_phase, + force_exit=True, + config=config, + ) + except Exception: + octobot_commons.logging.get_logger("OctoBot").debug( + "Failed to record CLI startup failure in node journal", + exc_info=True, + ) + + def start_octobot(args, default_config_file=None): logger = None + config = None try: if args.version: print(constants.LONG_VERSION) @@ -518,8 +547,6 @@ def start_octobot(args, default_config_file=None): if not config.is_loaded(): raise errors.ConfigError - octobot_community.ActivityMetrics.initialize_tracker(config) - # Handle utility methods before bot initializing if possible if args.encrypter: commands.exchange_keys_encrypter() @@ -531,6 +558,8 @@ def start_octobot(args, default_config_file=None): community_auth = None if args.backtesting else asyncio.run( _get_authenticated_community_if_possible(config, logger) ) + octobot_community.init_sentry_tracker() + journal_startup.initialize_journal(config) # Startup order matters: sync user and tentacles/community config must run before # profile activation. First boot (empty user/) has no profiles until tentacles @@ -604,20 +633,23 @@ def start_octobot(args, default_config_file=None): force_error_exit = False except errors.RemoteConfigError as err: logger.exception(err) + _record_cli_startup_failure(err, "remote_config", config) force_error_exit = True except errors.ConfigError as err: logger.error("OctoBot can't start without a valid " + common_constants.CONFIG_FILE + " configuration file.\nError: " + str(err) + "\nYou can use " + constants.DEFAULT_CONFIG_FILE + " as an example to fix it.") + _record_cli_startup_failure(err, "config", config) force_error_exit = True - except errors.NoProfileError: + except errors.NoProfileError as err: logger.error("Missing default profiles. OctoBot can't start without a valid default profile configuration. " "Please make sure that the {config.profiles_path} " f"folder is accessible. To reinstall default profiles, delete the " f"'{tentacles_manager_constants.TENTACLES_PATH}' " f"folder or start OctoBot with the following arguments: tentacles --install --all") + _record_cli_startup_failure(err, "no_profile", config) force_error_exit = True except ModuleNotFoundError as err: @@ -626,18 +658,21 @@ def start_octobot(args, default_config_file=None): "please use the following command:\nstart.py tentacles --install --all") else: logger.exception(err) + _record_cli_startup_failure(err, "module_not_found", config) force_error_exit = True - except errors.ConfigEvaluatorError: + except errors.ConfigEvaluatorError as err: logger.error("OctoBot can't start without a valid configuration file.\n" "This file is generated on tentacle " "installation using the following command:\nstart.py tentacles --install --all") + _record_cli_startup_failure(err, "config_evaluator", config) force_error_exit = True - except errors.ConfigTradingError: + except errors.ConfigTradingError as err: logger.error("OctoBot can't start without a valid configuration file.\n" "This file is generated on tentacle " "installation using the following command:\nstart.py tentacles --install --all") + _record_cli_startup_failure(err, "config_trading", config) force_error_exit = True if force_error_exit: octobot_community.flush_tracker() diff --git a/octobot/commands.py b/octobot/commands.py index 7bda986211..0e7f19cb60 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -38,6 +38,8 @@ import octobot.constants as constants import octobot.community.tentacles_packages as community_tentacles_packages import octobot.configuration_manager as configuration_manager +import octobot.enums as enums +import octobot.community.node_journal.lifecycle as journal_startup COMMANDS_LOGGER_NAME = "Commands" IGNORED_COMMAND_WHEN_RESTART = ["-u", "--update"] @@ -340,6 +342,18 @@ def run_bot(bot, logger): bot.task_manager.run_forever(start_bot(bot, logger)) +def _record_bot_initialize_failure(bot, error: Exception, *, catch: bool) -> None: + if configuration_manager.get_distribution(bot.config) is not enums.OctoBotDistribution.NODE: + return + edited_config = bot.get_edited_config(constants.CONFIG_KEY, dict_only=False) + journal_startup.record_node_startup_failed( + error, + startup_phase="bot_initialize", + force_exit=not catch, + config=edited_config, + ) + + async def start_bot(bot, logger, catch=False): try: # load tentacles details @@ -354,6 +368,7 @@ async def start_bot(bot, logger, catch=False): except Exception as e: logger.exception(e) + _record_bot_initialize_failure(bot, e, catch=catch) if not catch: raise stop_bot(bot) diff --git a/octobot/community/__init__.py b/octobot/community/__init__.py index 63be4d8210..a86e738c16 100644 --- a/octobot/community/__init__.py +++ b/octobot/community/__init__.py @@ -64,9 +64,6 @@ get_current_octobots_stats, can_read_metrics, ) -from octobot.community.activity_analysis.activity_metrics import ( - ActivityMetrics, -) from octobot.community.authentication import ( CommunityAuthentication, ) @@ -123,7 +120,6 @@ "get_community_metrics", "get_current_octobots_stats", "can_read_metrics", - "ActivityMetrics", "CommunityAuthentication", "CommunityTentaclesPackage", "CommunitySupports", diff --git a/octobot/community/activity_analysis/bot_id_resolver.py b/octobot/community/activity_analysis/bot_id_resolver.py deleted file mode 100644 index ec3387e489..0000000000 --- a/octobot/community/activity_analysis/bot_id_resolver.py +++ /dev/null @@ -1,50 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import dataclasses -import uuid - -import octobot_commons.configuration as configuration -import octobot_commons.constants as commons_constants - - -@dataclasses.dataclass(frozen=True) -class BotIdResolution: - bot_id: str - was_created: bool - - -def _get_metrics_section(config: configuration.Configuration) -> dict: - metrics_section = config.config.setdefault(commons_constants.CONFIG_METRICS, {}) - if not isinstance(metrics_section, dict): - raise ValueError(f"{commons_constants.CONFIG_METRICS} must be a mapping in config") - return metrics_section - - -def ensure_activity_bot_id(config: configuration.Configuration) -> BotIdResolution: - metrics_section = _get_metrics_section(config) - stored_bot_id = metrics_section.get(commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID) - if stored_bot_id: - return BotIdResolution( - bot_id=str(stored_bot_id), - was_created=False, - ) - new_bot_id = str(uuid.uuid4()) - metrics_section[commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = new_bot_id - config.save() - return BotIdResolution( - bot_id=new_bot_id, - was_created=True, - ) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index 4964ab2ef3..9de6db6ca1 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -20,8 +20,7 @@ import time import threading import typing -import octobot.community.activity_analysis.config_path_binding as config_path_binding -import octobot.community.activity_analysis.activity_metrics as activity_metrics +import octobot.community.config_path_binding as config_path_binding import decimal import octobot.constants as constants @@ -150,6 +149,7 @@ def create(configuration: commons_configuration.Configuration, **kwargs): ) def update(self, configuration: commons_configuration.Configuration): + self.config = configuration self.configuration_storage.set_configuration(configuration) self._wallet_backend = wallet_backend.WalletBackend( self._get_wallet_sync_storage(), self.logger @@ -583,7 +583,6 @@ def _clear_bot_scoped_config(self): "and webhook url will be different on this bot." ) self._save_bot_id("") - activity_metrics.ActivityMetrics.clear_activity_bot_id(self.config) self.save_tradingview_email("") # also reset mqtt id to force a new mqtt id creation self._save_mqtt_device_uuid("") @@ -723,6 +722,9 @@ def get_wallet(self, address: str) -> sync_chain.Wallet: def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: return self._wallet_backend.get_wallet_by_user_id(user_id) + def has_wallet_for_user_id(self, user_id: str) -> bool: + return self._wallet_backend.has_wallet_for_user_id(user_id) + async def get_session_for_address(self, address: str) -> starfish_spaces.Session: """Build (and cache) a dk-namespace starfish_spaces Session for the given wallet. diff --git a/octobot/community/community_analysis.py b/octobot/community/community_analysis.py index 4ae2dfd330..093e545c59 100644 --- a/octobot/community/community_analysis.py +++ b/octobot/community/community_analysis.py @@ -86,7 +86,7 @@ async def get_community_metrics(): def can_read_metrics(config): - return constants.IS_CLOUD_ENV or config.get_metrics_enabled() + return False def _format_top_elements(top_elements): diff --git a/octobot/community/activity_analysis/activity_metrics.py b/octobot/community/community_bot_stats.py similarity index 62% rename from octobot/community/activity_analysis/activity_metrics.py rename to octobot/community/community_bot_stats.py index 901bb93cfb..6a0563eda8 100644 --- a/octobot/community/activity_analysis/activity_metrics.py +++ b/octobot/community/community_bot_stats.py @@ -13,62 +13,30 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . + +"""Periodic community bot config/stats updates. + +``CommunityBotStats`` is temporary until full migration to OctoBot v3 replaces this path. +""" + import asyncio import octobot_commons.logging as logging -import octobot_commons.configuration as configuration import octobot_commons.authentication as authentication import octobot_commons.constants as common_constants import octobot_trading.api as trading_api -import octobot.community.activity_analysis.bot_id_resolver as bot_id_resolver -import octobot.community.errors_upload.sentry_tracker as tracker -import octobot.constants as constants -import octobot.enums as enums - -class ActivityMetrics: +class CommunityBotStats: def __init__(self, octobot_api): self.octobot_api = octobot_api - self.edited_config: configuration.Configuration = octobot_api.get_edited_config(dict_only=False) - self.enabled = constants.IS_CLOUD_ENV or self.edited_config.get_metrics_enabled() self.logger = logging.get_logger(self.__class__.__name__) self.keep_running = True - - @staticmethod - def initialize_tracker(config: configuration.Configuration) -> None: - tracker.init_sentry_tracker(metrics_enabled=config.get_metrics_enabled()) - - @staticmethod - def clear_activity_bot_id(config: configuration.Configuration) -> None: - metrics_section = config.config.setdefault(common_constants.CONFIG_METRICS, {}) - if isinstance(metrics_section, dict): - metrics_section[common_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = "" - - def setup_activity_tracking(self, distribution: enums.OctoBotDistribution) -> None: - if not self.enabled: - return - resolution = bot_id_resolver.ensure_activity_bot_id(self.edited_config) - if tracker.activity_tracking_is_active(): - tracker.update_tracker_bot_id(resolution.bot_id) - if distribution is enums.OctoBotDistribution.NODE and resolution.was_created: - tracker.track_usage_event( - "node_first_start", - distribution="node", - version=constants.LONG_VERSION, - ) - - @staticmethod - def report_child_octobot_first_start() -> None: - if not tracker.has_tracker_bot_id(): - return - tracker.track_usage_event("child_octobot_first_start") + self.task_enabled = False async def start_community_task(self): - if not self.enabled: - return try: while self.keep_running: await asyncio.sleep(common_constants.TIMER_BETWEEN_METRICS_UPTIME_UPDATE) diff --git a/octobot/community/activity_analysis/config_path_binding.py b/octobot/community/config_path_binding.py similarity index 99% rename from octobot/community/activity_analysis/config_path_binding.py rename to octobot/community/config_path_binding.py index e97ebf3198..da8789baf5 100644 --- a/octobot/community/activity_analysis/config_path_binding.py +++ b/octobot/community/config_path_binding.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . + import dataclasses import hashlib import os diff --git a/octobot/community/errors_upload/__init__.py b/octobot/community/errors_upload/__init__.py index 6af2d0b4a7..725fb45502 100644 --- a/octobot/community/errors_upload/__init__.py +++ b/octobot/community/errors_upload/__init__.py @@ -1,25 +1,10 @@ # This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) # Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . from octobot.community.errors_upload import sentry_tracker from octobot.community.errors_upload.sentry_tracker import ( - init_sentry_tracker, flush_tracker, - update_tracker_bot_id, - track_usage_event, + init_sentry_tracker, ) __all__ = [ diff --git a/octobot/community/errors_upload/sentry_tracker.py b/octobot/community/errors_upload/sentry_tracker.py index 1e015ad107..7155176bb4 100644 --- a/octobot/community/errors_upload/sentry_tracker.py +++ b/octobot/community/errors_upload/sentry_tracker.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import time -import typing import sentry_sdk @@ -25,72 +24,50 @@ _sentry_initialized = False -_activity_tracking_active = False -_tracker_bot_id_set: bool = False -def init_sentry_tracker(metrics_enabled: bool) -> None: - """ - Initialize Sentry when ACTIVITY_TRACKER_DSN (metrics enabled) or ERROR_TRACKER_DSN is set. - Tracker DSN takes priority when metrics are enabled. - """ - global _activity_tracking_active, _sentry_initialized +def init_sentry_tracker() -> None: + """Initialize Sentry error reporting when ERROR_TRACKER_DSN is set.""" + global _sentry_initialized logger = octobot_commons.logging.get_logger("sentry_tracker") - activity_dsn = octobot.constants.ACTIVITY_TRACKER_DSN error_dsn = octobot.constants.ERROR_TRACKER_DSN - use_activity_dsn = bool(activity_dsn and metrics_enabled) - active_dsn = activity_dsn if use_activity_dsn else error_dsn - - if not active_dsn: - logger.debug("Sentry tracker disabled: no applicable DSN") - _activity_tracking_active = False + if not error_dsn: + logger.debug("Sentry tracker disabled: ERROR_TRACKER_DSN is not set") _sentry_initialized = False return if _sentry_initialized: - _activity_tracking_active = use_activity_dsn return environment = "cloud" if octobot.constants.IS_CLOUD_ENV else "self hosted" app_name = f"{octobot.constants.PROJECT_NAME} open source" - init_kwargs: dict[str, typing.Any] = { - "dsn": active_dsn, + # Error DSN: auto-capture log errors; strip bot log prefix before upload. + sentry_sdk.init( + dsn=error_dsn, # Percent of error events to send to the server. # 0.5 would be 50%. Defaults to 1.0. - "sample_rate": 1, + sample_rate=1, # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # We recommend adjusting this value in production. - "traces_sample_rate": 0, + traces_sample_rate=0, # By default the SDK will try to use the SENTRY_RELEASE # environment variable, or infer a git commit # SHA as release, however you may want to set # something more human-readable. - "release": octobot.constants.LONG_VERSION, - "include_local_variables": False, # careful not to upload sensitive data + release=octobot.constants.LONG_VERSION, + include_local_variables=False, # careful not to upload sensitive data # breadcrumbs from multiple tasks and activities are mixed, only take the last ones - "max_breadcrumbs": 5, - "environment": environment, - } - if use_activity_dsn: - init_kwargs["sample_rate"] = 0 - # Activity-only: no default integrations (especially LoggingIntegration). - # Only explicit track_usage_event calls send data. - init_kwargs["default_integrations"] = False - init_kwargs["integrations"] = [] - else: - # Error DSN: auto-capture log errors; strip bot log prefix before upload. - init_kwargs["before_send"] = _before_send - - sentry_sdk.init(**init_kwargs) - _activity_tracking_active = use_activity_dsn + max_breadcrumbs=5, + environment=environment, + before_send=_before_send, + ) _sentry_initialized = True - mode = "activity" if use_activity_dsn else "error" logger.info( - f"Initialized {mode} tracking with environment: {environment}, " - f"release: {octobot.constants.LONG_VERSION}, dsn: {active_dsn}" + f"Initialized error tracking with environment: {environment}, " + f"release: {octobot.constants.LONG_VERSION}, dsn: {error_dsn}" ) sentry_sdk.set_context("app", { "app_start_time": str(round(time.time())), @@ -110,34 +87,6 @@ def flush_tracker() -> None: time.sleep(delay) -def activity_tracking_is_active() -> bool: - return _activity_tracking_active - - -def has_tracker_bot_id() -> bool: - return _tracker_bot_id_set - - -def update_tracker_bot_id(bot_id: str) -> None: - global _tracker_bot_id_set - _tracker_bot_id_set = True - sentry_sdk.set_user({"id": bot_id}) - sentry_sdk.set_tag("bot_id", bot_id) - - -def track_usage_event(event_name: str, **attributes: typing.Any) -> None: - metric_attributes = {"event": event_name} - for attribute_key, attribute_value in attributes.items(): - if attribute_value is not None: - metric_attributes[attribute_key] = str(attribute_value) - sentry_sdk.metrics.count("octobot.usage", 1, attributes=metric_attributes) - octobot_commons.logging.get_logger("sentry_tracker").debug( - "Tracked usage event %s with attributes %s", - event_name, - metric_attributes, - ) - - def _get_log_prefix() -> str: return f"[{octobot.constants.COMMUNITY_BOT_ID}] " if octobot.constants.COMMUNITY_BOT_ID else "[self-hosted]" diff --git a/octobot/community/node_journal/__init__.py b/octobot/community/node_journal/__init__.py new file mode 100644 index 0000000000..57072fa51e --- /dev/null +++ b/octobot/community/node_journal/__init__.py @@ -0,0 +1,127 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Public node journal API. + +Import tiers: +- Tier 1 (feature code): ``octobot.community.node_journal`` — record/read helpers and events. +- Tier 2 (infrastructure): ``octobot.community.node_journal.lifecycle``, + ``octobot.community.node_journal.recording_context``. +- Tier 3 (internal/tests): ``octobot.community.node_journal.store``, + ``octobot.community.node_journal.state``. + +Layer rules (documentation only): +- ``journal``, ``store``, ``state`` must not import from ``recording/`` or ``lifecycle``. +- ``recording/*`` may import ``journal``, ``state``, and ``classify``; not ``lifecycle``. +- ``lifecycle`` may import ``journal``, ``recording``, and external infrastructure. +- ``journey_summary`` is read-only; no ``recording`` imports. +""" + +from octobot.community.node_journal.events import ( + NodeJournalEvent, + UI_JOURNAL_EVENTS, +) +from octobot.community.node_journal.journal import ( + initialize_for_config, + is_journal_enabled, + read_events, + record, + record_failure, +) +from octobot.community.node_journal.journey_summary import ( + build_journey_summary, + build_upload_envelope, +) +import octobot.community.node_journal.recording as journal_recording + +on_user_data_pull_succeeded = journal_recording.on_user_data_pull_succeeded +record_account_auth_create_succeeded = journal_recording.record_account_auth_create_succeeded +record_account_auth_deleted = journal_recording.record_account_auth_deleted +record_account_deleted = journal_recording.record_account_deleted +record_account_edit_succeeded = journal_recording.record_account_edit_succeeded +record_account_validated = journal_recording.record_account_validated +record_account_validated_from_account = journal_recording.record_account_validated_from_account +record_account_validation_failed = journal_recording.record_account_validation_failed +record_accounts_refreshed = journal_recording.record_accounts_refreshed +record_automation_restarted = journal_recording.record_automation_restarted +record_automation_run_errored = journal_recording.record_automation_run_errored +record_automation_started = journal_recording.record_automation_started +record_automation_stopped = journal_recording.record_automation_stopped +record_executor_failure = journal_recording.record_executor_failure +record_existing_config_detected = journal_recording.record_existing_config_detected +record_external_action_failed = journal_recording.record_external_action_failed +record_external_action_received = journal_recording.record_external_action_received +record_first_automation_started = journal_recording.record_first_automation_started +record_new_automation_created = journal_recording.record_new_automation_created +record_new_automation_created_from_strategy = journal_recording.record_new_automation_created_from_strategy +record_process_startup_failed = journal_recording.record_process_startup_failed +record_process_startup_succeeded = journal_recording.record_process_startup_succeeded +record_reconcile_completed = journal_recording.record_reconcile_completed +record_scheduler_init_failed = journal_recording.record_scheduler_init_failed +record_strategy_create_succeeded = journal_recording.record_strategy_create_succeeded +record_strategy_edit_succeeded = journal_recording.record_strategy_edit_succeeded +record_sync_read_failed = journal_recording.record_sync_read_failed +record_sync_storage_event = journal_recording.record_sync_storage_event +record_wallet_operation_failed = journal_recording.record_wallet_operation_failed +record_wallet_setup_attempt = journal_recording.record_wallet_setup_attempt +record_wallet_setup_failed = journal_recording.record_wallet_setup_failed +record_wallet_setup_succeeded = journal_recording.record_wallet_setup_succeeded +reset_sync_tracker_after_node_startup = journal_recording.reset_sync_tracker_after_node_startup + +__all__ = [ + "NodeJournalEvent", + "UI_JOURNAL_EVENTS", + "build_journey_summary", + "build_upload_envelope", + "initialize_for_config", + "is_journal_enabled", + "on_user_data_pull_succeeded", + "read_events", + "record", + "record_account_auth_create_succeeded", + "record_account_auth_deleted", + "record_account_deleted", + "record_account_edit_succeeded", + "record_account_validated", + "record_account_validated_from_account", + "record_account_validation_failed", + "record_accounts_refreshed", + "record_automation_restarted", + "record_automation_run_errored", + "record_automation_started", + "record_automation_stopped", + "record_executor_failure", + "record_existing_config_detected", + "record_external_action_failed", + "record_external_action_received", + "record_failure", + "record_first_automation_started", + "record_new_automation_created", + "record_new_automation_created_from_strategy", + "record_strategy_create_succeeded", + "record_strategy_edit_succeeded", + "record_process_startup_failed", + "record_process_startup_succeeded", + "record_reconcile_completed", + "record_scheduler_init_failed", + "record_sync_read_failed", + "record_sync_storage_event", + "record_wallet_operation_failed", + "record_wallet_setup_attempt", + "record_wallet_setup_failed", + "record_wallet_setup_succeeded", + "reset_sync_tracker_after_node_startup", +] diff --git a/octobot/community/node_journal/constants.py b/octobot/community/node_journal/constants.py new file mode 100644 index 0000000000..3b510fff11 --- /dev/null +++ b/octobot/community/node_journal/constants.py @@ -0,0 +1,36 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums + +JOURNAL_MAX_EVENTS = 50_000 +JOURNAL_SCHEMA_VERSION = 1 +SYNC_SESSION_GAP_SECONDS = 14_400 +JOURNAL_DIR_NAME = "node_journal" +MANIFEST_FILE_NAME = "manifest.json" +STORAGE_CTX_KEY = "_ctx" +EVENTS_FILE_NAME = "events.jsonl" +ONBOARDING_SEGMENT_FILE_NAME = "onboarding_segment.jsonl" +CONFIG_JOURNAL_SECTION = "journal" +CONFIG_INSTALL_ID = journal_enums.JournalManifestField.INSTALL_ID.value +CONFIG_ONBOARDING_STARTED_AT = "onboarding_started_at" +CONFIG_FIRST_AUTOMATION_STARTED_AT = "first_automation_started_at" +CONFIG_CONNECTION_SEQUENCE = "connection_sequence" +CONFIG_LAST_USER_DATA_PULL_AT = "last_user_data_pull_at" +CONFIG_TRACKED_AUTOMATION_IDS = "tracked_automation_ids" +DISTRIBUTION_NODE = "node" +JOURNAL_ENABLED_ENV_VAR = "OCTOBOT_NODE_JOURNAL_ENABLED" +JOURNAL_ENABLED_DEFAULT = "true" diff --git a/octobot/community/node_journal/enums.py b/octobot/community/node_journal/enums.py new file mode 100644 index 0000000000..bf56cca6fa --- /dev/null +++ b/octobot/community/node_journal/enums.py @@ -0,0 +1,121 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +class JournalEventLineField(enum.StrEnum): + EVENT = "event" + TIMESTAMP = "timestamp" + SESSION_ID = "session_id" + INSTALL_ID = "install_id" + APP_VERSION = "app_version" + DISTRIBUTION = "distribution" + ONBOARDING_COMPLETE = "onboarding_complete" + ATTRIBUTES = "attributes" + RECORDED = "recorded" + + +class JournalStorageContextField(enum.StrEnum): + SESSION_ID = "session_id" + APP_VERSION = "app_version" + + +class JournalManifestField(enum.StrEnum): + SCHEMA = "schema" + INSTALL_ID = "install_id" + + +class JournalInitPhase(enum.StrEnum): + DBOS_CREATE = "dbos_create" + REGISTER_WORKFLOWS = "register_workflows" + VERSION_MIGRATION = "version_migration" + DBOS_LAUNCH = "dbos_launch" + REGISTER_SCHEDULES = "register_schedules" + + +class JournalStartupPhase(enum.StrEnum): + NODE_API_START = "node_api_start" + BOT_INITIALIZE = "bot_initialize" + CLI_START = "cli_start" + + +class JournalSchedulerBackend(enum.StrEnum): + POSTGRES = "postgres" + SQLITE = "sqlite" + + +class WalletSetupFailureReason(enum.StrEnum): + SERVICE_UNAVAILABLE = "service_unavailable" + ALREADY_CONFIGURED = "already_configured" + CONCURRENT_RACE = "concurrent_race" + WALLET_ERROR = "wallet_error" + + +class WalletSetupMethod(enum.StrEnum): + CREATE = "create" + IMPORT = "import" + + +class WalletOperation(enum.StrEnum): + IMPORT = "import" + CREATE = "create" + DECRYPT = "decrypt" + LOOKUP = "lookup" + DELETE = "delete" + RENAME = "rename" + + +class SyncReadFailureReason(enum.StrEnum): + CAP_AUTH = "cap_auth" + WALLET_NOT_FOUND = "wallet_not_found" + IDENTITY_MISSING = "identity_missing" + STORAGE_ERROR = "storage_error" + OTHER = "other" + + +class SyncStorageRecoveryAction(enum.StrEnum): + DROP_INVALID_ITEMS = "drop_invalid_items" + SANITIZE_STATE = "sanitize_state" + SKIP_ITEM = "skip_item" + REBUILD_ITEM = "rebuild_item" + FALLBACK_ITEM = "fallback_item" + + +class SyncStorageProvider(enum.StrEnum): + LOCAL = "local" + + +class ConfigurationType(enum.StrEnum): + GENERIC_PROCESS = "generic_process" + MARKET_MAKING = "market_making" + COPY = "copy" + SIGNAL_BOT = "signal_bot" + GENERIC_WORKFLOW = "generic_workflow" + TRADING_TENTACLES = "trading_tentacles" + + +class OctobotKind(enum.StrEnum): + MANUAL = "manual" + MARKET_MAKING = "market_making" + FLOW = "flow" + + +class FlowSubtype(enum.StrEnum): + COPY = "copy" + SIGNAL_BOT = "signal_bot" + AI_AGENTS = "ai_agents" + OTHER = "other" diff --git a/octobot/community/node_journal/events.py b/octobot/community/node_journal/events.py new file mode 100644 index 0000000000..f54694bbbb --- /dev/null +++ b/octobot/community/node_journal/events.py @@ -0,0 +1,105 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +class NodeJournalEvent(enum.StrEnum): + UNKNOWN = "unknown" + NODE_PROCESS_STARTUP_SUCCEEDED = "node_process_startup_succeeded" + NODE_PROCESS_STARTUP_FAILED = "node_process_startup_failed" + WALLET_SETUP_ATTEMPT = "wallet_setup_attempt" + WALLET_SETUP_SUCCEEDED = "wallet_setup_succeeded" + WALLET_SETUP_FAILED = "wallet_setup_failed" + WALLET_OPERATION_FAILED = "wallet_operation_failed" + EXTERNAL_INTERFACE_CONNECTED = "external_interface_connected" + SYNC_READ_FAILED = "sync_read_failed" + EXTERNAL_ACTION_RECEIVED = "external_action_received" + EXTERNAL_ACTION_FAILED = "external_action_failed" + ACCOUNT_AUTH_CREATE_ATTEMPT = "account_auth_create_attempt" + ACCOUNT_AUTH_CREATE_SUCCEEDED = "account_auth_create_succeeded" + ACCOUNT_AUTH_CREATE_FAILED = "account_auth_create_failed" + ACCOUNT_CREATE_ATTEMPT = "account_create_attempt" + ACCOUNT_VALIDATED = "account_validated" + ACCOUNT_VALIDATION_FAILED = "account_validation_failed" + ACCOUNT_EDIT_ATTEMPT = "account_edit_attempt" + ACCOUNT_EDIT_SUCCEEDED = "account_edit_succeeded" + ACCOUNT_EDIT_FAILED = "account_edit_failed" + STRATEGY_CREATE_ATTEMPT = "strategy_create_attempt" + STRATEGY_CREATE_SUCCEEDED = "strategy_create_succeeded" + STRATEGY_CREATE_FAILED = "strategy_create_failed" + STRATEGY_EDIT_ATTEMPT = "strategy_edit_attempt" + STRATEGY_EDIT_SUCCEEDED = "strategy_edit_succeeded" + STRATEGY_EDIT_FAILED = "strategy_edit_failed" + AUTOMATION_CREATE_ATTEMPT = "automation_create_attempt" + AUTOMATION_CREATE_FAILED = "automation_create_failed" + FIRST_AUTOMATION_STARTED = "first_automation_started" + AUTOMATION_EDIT_ATTEMPT = "automation_edit_attempt" + AUTOMATION_EDIT_SUCCEEDED = "automation_edit_succeeded" + AUTOMATION_EDIT_FAILED = "automation_edit_failed" + AUTOMATION_STARTED = "automation_started" + AUTOMATION_STOPPED = "automation_stopped" + AUTOMATION_RESTARTED = "automation_restarted" + AUTOMATION_RUN_ERRORED = "automation_run_errored" + ACCOUNT_DELETED = "account_deleted" + ACCOUNT_AUTH_DELETED = "account_auth_deleted" + ACCOUNTS_REFRESHED = "accounts_refreshed" + EXISTING_CONFIG_DETECTED = "existing_config_detected" + RECONCILE_COMPLETED = "reconcile_completed" + UI_BOOT_FAILED = "ui_boot_failed" + UI_AUTH_STATE_BROKEN = "ui_auth_state_broken" + UI_FATAL_RENDER_ERROR = "ui_fatal_render_error" + UI_CLIENT_STORAGE_RESET = "ui_client_storage_reset" + UI_INSECURE_CONTEXT = "ui_insecure_context" + SYNC_STORAGE_DECRYPT_FAILED = "sync_storage_decrypt_failed" + SYNC_STORAGE_FORMAT_ERROR = "sync_storage_format_error" + SYNC_STORAGE_SCHEMA_RECOVERY = "sync_storage_schema_recovery" + SYNC_STORAGE_LOAD_FAILED = "sync_storage_load_failed" + SCHEDULER_INIT_FAILED = "scheduler_init_failed" + + +# Must match node_web_interface UiJournalEvent union (5 client-emitted events). +UI_JOURNAL_EVENTS = frozenset({ + NodeJournalEvent.UI_BOOT_FAILED, + NodeJournalEvent.UI_AUTH_STATE_BROKEN, + NodeJournalEvent.UI_FATAL_RENDER_ERROR, + NodeJournalEvent.UI_CLIENT_STORAGE_RESET, + NodeJournalEvent.UI_INSECURE_CONTEXT, +}) + +UI_BLOCKING_JOURNAL_EVENTS = frozenset( + event for event in UI_JOURNAL_EVENTS if event != NodeJournalEvent.UI_CLIENT_STORAGE_RESET +) + +FAILURE_EVENTS = frozenset( + event + for event in NodeJournalEvent + if event.value.endswith("_failed") or event.value.endswith("_errored") +) + + +def coerce_node_journal_event(raw_event: NodeJournalEvent | str) -> tuple[NodeJournalEvent, str | None]: + if isinstance(raw_event, NodeJournalEvent): + if raw_event == NodeJournalEvent.UNKNOWN: + return NodeJournalEvent.UNKNOWN, None + return raw_event, None + event_name = str(raw_event) + if event_name in NodeJournalEvent._value2member_map_: + parsed_event = NodeJournalEvent(event_name) + if parsed_event == NodeJournalEvent.UNKNOWN: + return NodeJournalEvent.UNKNOWN, None + return parsed_event, None + return NodeJournalEvent.UNKNOWN, event_name diff --git a/octobot/community/node_journal/events_metadata.py b/octobot/community/node_journal/events_metadata.py new file mode 100644 index 0000000000..36b1572f70 --- /dev/null +++ b/octobot/community/node_journal/events_metadata.py @@ -0,0 +1,64 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.events as journal_events + +FUNNEL_STEP_ORDER: tuple[journal_events.NodeJournalEvent, ...] = ( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_SUCCEEDED, + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_FAILED, + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + journal_events.NodeJournalEvent.WALLET_SETUP_FAILED, + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_FAILED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATION_FAILED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_FAILED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_FAILED, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_FAILED, + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, +) + +FUNNEL_STEP_RANK = {event: index for index, event in enumerate(FUNNEL_STEP_ORDER)} + +JOURNEY_MILESTONE_LABELS: dict[journal_events.NodeJournalEvent, str] = { + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED: "wallet_setup", + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED: "external_connect", + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED: "account_validated", + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED: "strategy_create", + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED: "strategy_edit", + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED: "first_automation", +} + +JOURNEY_MILESTONE_LABEL_ORDER: tuple[str, ...] = tuple( + JOURNEY_MILESTONE_LABELS[event] + for event in FUNNEL_STEP_ORDER + if event in JOURNEY_MILESTONE_LABELS +) + +JOURNEY_SUCCESS_EVENTS = frozenset({ + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, +}) diff --git a/octobot/community/node_journal/journal.py b/octobot/community/node_journal/journal.py new file mode 100644 index 0000000000..51985c37c2 --- /dev/null +++ b/octobot/community/node_journal/journal.py @@ -0,0 +1,211 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Core write/read pipeline for the node journal.""" + +import logging +import time +import typing + +import octobot.constants as octobot_constants +import octobot_commons.os_util as os_util + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state +import octobot.community.node_journal.store as journal_store + +logger = logging.getLogger(__name__) + +def is_journal_enabled() -> bool: + return os_util.parse_boolean_environment_var( + journal_constants.JOURNAL_ENABLED_ENV_VAR, + journal_constants.JOURNAL_ENABLED_DEFAULT, + ) + + +def run_journal_operation( + operation_name: str, + operation: typing.Callable[[], typing.Any], + *, + default: typing.Any, +) -> typing.Any: + try: + return operation() + except Exception as exc: + logger.exception("Journal %s failed: %s", operation_name, exc) + return default + + +def sanitize_attribute_value(value: typing.Any) -> typing.Any: + if isinstance(value, bool): + return value + if isinstance(value, (int, float, str)): + return value + if isinstance(value, list): + return value + return str(value) + + +def sanitize_event_attributes( + event: journal_events.NodeJournalEvent, + attributes: journal_models.JournalEventAttributes, +) -> journal_models.JournalEventAttributes: + sanitized_attributes = {} + for field_name, value in attributes.to_dict().items(): + if value is None: + continue + sanitized_attributes[field_name] = sanitize_attribute_value(value) + return journal_models.JournalEventAttributes.from_dict(sanitized_attributes) + + +def record( + event: journal_events.NodeJournalEvent | str, + *, + attributes: journal_models.JournalEventAttributes | dict | None = None, + timestamp: float | None = None, +) -> journal_models.JournalEventLine: + return run_journal_operation( + "record", + lambda: _record(event, attributes=attributes, timestamp=timestamp), + default=_build_disabled_event_stub_for_input(event, attributes), + ) + + +def read_events() -> list[journal_models.JournalEventLine]: + if not is_journal_enabled(): + return [] + return journal_store.get_store().read_all_events() + + +def initialize_for_config(config) -> None: + if not is_journal_enabled(): + return + journal_state.bind_config(config) + journal_state.load_persisted_state(config) + + +def record_failure( + event: journal_events.NodeJournalEvent, + *, + error: BaseException | None = None, + error_message: str | None = None, + error_category: str | None = None, + attributes: journal_models.JournalEventAttributes | dict | None = None, +) -> journal_models.JournalEventLine: + failure_attributes = journal_models.JournalEventAttributes.merge( + _coerce_attributes(attributes), + {}, + ) + if error_category is None and error is not None: + failure_attributes.error_category = error.__class__.__name__ + elif error_category is not None: + failure_attributes.error_category = error_category + if error_message is not None: + failure_attributes.error_message = error_message + return record(event, attributes=failure_attributes) + + +def _record( + event: journal_events.NodeJournalEvent | str, + *, + attributes: journal_models.JournalEventAttributes | dict | None, + timestamp: float | None, +) -> journal_models.JournalEventLine: + parsed_event, raw_event_name = journal_events.coerce_node_journal_event(event) + coerced_attributes = _merge_raw_event_name(_coerce_attributes(attributes), raw_event_name) + if parsed_event == journal_events.NodeJournalEvent.UNKNOWN: + logger.error("Unknown journal event: %s", raw_event_name or event) + return _build_disabled_event_stub(parsed_event, coerced_attributes, recorded=False) + if not is_journal_enabled(): + return _build_disabled_event_stub(parsed_event, coerced_attributes) + sanitized_attributes = sanitize_event_attributes(parsed_event, coerced_attributes) + if ( + parsed_event in journal_events.FAILURE_EVENTS + and sanitized_attributes.error_category is None + ): + logger.error("%s requires error_category", parsed_event.value) + return _build_disabled_event_stub(parsed_event, sanitized_attributes, recorded=False) + emit_timestamp = time.time() if timestamp is None else timestamp + persisted_state = journal_state.load_persisted_state() + event_line = journal_models.JournalEventLine( + event=parsed_event, + timestamp=emit_timestamp, + session_id=journal_state.get_session_id(), + install_id=persisted_state.install_id, + app_version=octobot_constants.LONG_VERSION, + distribution=journal_constants.DISTRIBUTION_NODE, + onboarding_complete=persisted_state.onboarding_complete, + attributes=sanitized_attributes, + ) + is_onboarding_segment = not persisted_state.onboarding_complete + journal_store.get_store().append(event_line, is_onboarding_segment=is_onboarding_segment) + if parsed_event == journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED: + journal_state.mark_first_automation_started(emit_timestamp) + return event_line + + +def _coerce_attributes( + attributes: journal_models.JournalEventAttributes | dict | None, +) -> journal_models.JournalEventAttributes: + if attributes is None: + return journal_models.JournalEventAttributes() + if isinstance(attributes, journal_models.JournalEventAttributes): + return attributes + return journal_models.JournalEventAttributes.from_dict(attributes) + + +def _merge_raw_event_name( + attributes: journal_models.JournalEventAttributes, + raw_event_name: str | None, +) -> journal_models.JournalEventAttributes: + if raw_event_name is None: + return attributes + return journal_models.JournalEventAttributes.merge( + attributes, + {"raw_event_name": raw_event_name}, + ) + + +def _build_disabled_event_stub_for_input( + event: journal_events.NodeJournalEvent | str, + attributes: journal_models.JournalEventAttributes | dict | None, + *, + recorded: bool = False, +) -> journal_models.JournalEventLine: + parsed_event, raw_event_name = journal_events.coerce_node_journal_event(event) + coerced_attributes = _merge_raw_event_name(_coerce_attributes(attributes), raw_event_name) + return _build_disabled_event_stub(parsed_event, coerced_attributes, recorded=recorded) + + +def _build_disabled_event_stub( + parsed_event: journal_events.NodeJournalEvent, + attributes: journal_models.JournalEventAttributes, + *, + recorded: bool = False, +) -> journal_models.JournalEventLine: + return journal_models.JournalEventLine( + event=parsed_event, + timestamp=time.time(), + session_id="", + install_id="", + app_version=octobot_constants.LONG_VERSION, + distribution=journal_constants.DISTRIBUTION_NODE, + onboarding_complete=False, + attributes=attributes, + recorded=recorded, + ) diff --git a/octobot/community/node_journal/journey_summary.py b/octobot/community/node_journal/journey_summary.py new file mode 100644 index 0000000000..4bd2b82b20 --- /dev/null +++ b/octobot/community/node_journal/journey_summary.py @@ -0,0 +1,206 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import collections + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.events_metadata as journal_events_metadata +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state + +def build_journey_summary(events: list[journal_models.JournalEventLine]) -> journal_models.JourneySummary: + persisted_state = journal_state.load_persisted_state() + install_start = persisted_state.onboarding_started_at + parsed_events = _parse_events(events) + onboarding_complete = persisted_state.onboarding_complete or any( + event == journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED for event, _ in parsed_events + ) + external_stats = _get_external_connect_stats(parsed_events) + return journal_models.JourneySummary( + onboarding_complete=onboarding_complete, + furthest_step_reached=_get_furthest_step(parsed_events), + last_successful_step=_get_last_successful_step(parsed_events), + first_failure=_get_first_failure(parsed_events), + retry_counts=_get_retry_counts(parsed_events), + step_durations_seconds=_get_step_durations(parsed_events, install_start), + step_deltas_seconds=_get_step_deltas(_get_step_durations(parsed_events, install_start)), + external_connect_count=external_stats["external_connect_count"], + first_external_connect_at=external_stats["first_external_connect_at"], + last_external_connect_at=external_stats["last_external_connect_at"], + longest_connect_gap_seconds=external_stats["longest_connect_gap_seconds"], + ui_blocking_issues_count=_get_ui_blocking_issues_count(parsed_events), + ) + + +def _parse_events( + events: list[journal_models.JournalEventLine], +) -> list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]]: + parsed_events = [] + for event_line in events: + if event_line.event == journal_events.NodeJournalEvent.UNKNOWN: + parsed_events.append((None, event_line)) + continue + parsed_events.append((event_line.event, event_line)) + return parsed_events + + +def _get_furthest_step( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> str | None: + best_rank = -1 + best_event_name = None + for event, _ in parsed_events: + if event is None: + continue + rank = journal_events_metadata.FUNNEL_STEP_RANK.get(event) + if rank is not None and rank >= best_rank: + best_rank = rank + best_event_name = event.value + return best_event_name + + +def _get_last_successful_step( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> str | None: + last_success = None + for event, _ in parsed_events: + if event in journal_events_metadata.JOURNEY_SUCCESS_EVENTS: + last_success = event.value + return last_success + + +def _get_first_failure( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> journal_models.FirstFailureInfo | None: + for event, event_line in parsed_events: + if event is None or event not in journal_events.FAILURE_EVENTS: + continue + attributes = event_line.attributes + return journal_models.FirstFailureInfo( + event=event.value, + timestamp=event_line.timestamp, + error_category=attributes.error_category, + error_message=attributes.error_message, + ) + return None + + +def _get_ui_blocking_issues_count( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> int: + blocking_issue_count = 0 + for event, _ in parsed_events: + if event is not None and event in journal_events.UI_BLOCKING_JOURNAL_EVENTS: + blocking_issue_count += 1 + return blocking_issue_count + + +def _get_retry_counts( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> dict[str, int]: + retry_counts: dict[str, int] = collections.Counter() + for event, _ in parsed_events: + if event is not None and event in journal_events.FAILURE_EVENTS: + retry_counts[event.value] += 1 + return dict(retry_counts) + + +def _get_step_durations( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], + install_start: float | None, +) -> dict[str, float]: + if install_start is None: + return {} + step_durations: dict[str, float] = {} + for event, event_line in parsed_events: + label = journal_events_metadata.JOURNEY_MILESTONE_LABELS.get(event) if event is not None else None + if label is None: + continue + event_timestamp = float(event_line.timestamp) + step_durations[label] = round(max(0.0, event_timestamp - install_start), 3) + return step_durations + + +def _get_step_deltas(step_durations_seconds: dict[str, float]) -> dict[str, float]: + ordered_labels = journal_events_metadata.JOURNEY_MILESTONE_LABEL_ORDER + deltas: dict[str, float] = {} + previous_label = None + previous_duration = None + for label in ordered_labels: + current_duration = step_durations_seconds.get(label) + if current_duration is None: + continue + if previous_label is not None and previous_duration is not None: + delta_key = f"{previous_label}_to_{label}" + deltas[delta_key] = round(max(0.0, current_duration - previous_duration), 3) + previous_label = label + previous_duration = current_duration + return deltas + + +def _get_external_connect_stats( + parsed_events: list[tuple[journal_events.NodeJournalEvent | None, journal_models.JournalEventLine]], +) -> dict: + connect_events = [ + event_line + for event, event_line in parsed_events + if event == journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED + ] + if not connect_events: + return { + "external_connect_count": 0, + "first_external_connect_at": None, + "last_external_connect_at": None, + "longest_connect_gap_seconds": None, + } + timestamps = [float(event_line.timestamp) for event_line in connect_events] + prior_gaps = [ + float(event_line.attributes.prior_gap_seconds) + for event_line in connect_events + if event_line.attributes.prior_gap_seconds is not None + ] + return { + "external_connect_count": len(connect_events), + "first_external_connect_at": min(timestamps), + "last_external_connect_at": max(timestamps), + "longest_connect_gap_seconds": max(prior_gaps) if prior_gaps else None, + } + + +def build_upload_envelope( + events: list[journal_models.JournalEventLine], + *, + app_version: str, + note: str | None = None, + ui_error_name: str | None = None, + ui_error_route: str | None = None, +) -> journal_models.UploadEnvelope: + persisted_state = journal_state.load_persisted_state() + journey_summary = build_journey_summary(events) + return journal_models.UploadEnvelope( + install_id=persisted_state.install_id, + app_version=app_version, + onboarding_started_at=persisted_state.onboarding_started_at, + onboarding_complete=journey_summary.onboarding_complete, + journey_summary=journey_summary, + events=events, + uploaded=False, + ready=True, + event_count=len(events), + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) diff --git a/octobot/community/node_journal/lifecycle.py b/octobot/community/node_journal/lifecycle.py new file mode 100644 index 0000000000..d68f3ddc47 --- /dev/null +++ b/octobot/community/node_journal/lifecycle.py @@ -0,0 +1,154 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import typing + +import octobot_commons.configuration as configuration + +import octobot.configuration_manager as configuration_manager +import octobot.enums as octobot_enums +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot.community.authentication as community_authentication +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.recording as journal_recording +import octobot.community.node_journal.state as journal_state + + +@dataclasses.dataclass(frozen=True) +class ExistingConfigSnapshot: + wallet_configured: bool + account_count: int + reconciled: bool + + +_reconcile_completed = False + + +def build_existing_config_snapshot() -> ExistingConfigSnapshot: + auth = community_authentication.CommunityAuthentication.instance() + wallet_configured = auth is not None and auth.is_node_wallet_configured() + account_count = 0 + if wallet_configured: + wallet_user_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() + for wallet_id in wallet_user_ids: + account_count += len(collection_providers.AccountProvider.instance().list_accounts(wallet_id)) + has_existing_config = wallet_configured or account_count > 0 + return ExistingConfigSnapshot( + wallet_configured=wallet_configured, + account_count=account_count, + reconciled=has_existing_config, + ) + + +def initialize_journal(config: configuration.Configuration) -> None: + if not journal_module.is_journal_enabled(): + return + distribution = configuration_manager.get_distribution(config.config) + if distribution is not octobot_enums.OctoBotDistribution.NODE: + return + journal_module.initialize_for_config(config) + snapshot = build_existing_config_snapshot() + if snapshot.reconciled: + journal_recording.record_existing_config_detected( + wallet_configured=snapshot.wallet_configured, + account_count=snapshot.account_count, + ) + + +def record_node_startup_succeeded(config: configuration.Configuration) -> None: + journal_module.initialize_for_config(config) + snapshot = build_existing_config_snapshot() + journal_recording.record_process_startup_succeeded( + wallet_configured=snapshot.wallet_configured, + new_install=not snapshot.reconciled, + reconciled=snapshot.reconciled, + ) + + +def record_node_startup_failed( + error: BaseException, + *, + startup_phase: journal_enums.JournalStartupPhase, + force_exit: bool, + config: typing.Optional[configuration.Configuration] = None, +) -> None: + resolved_config = config if config is not None else journal_state.get_config() + if resolved_config is not None: + journal_module.initialize_for_config(resolved_config) + snapshot = build_existing_config_snapshot() + journal_recording.record_process_startup_failed( + startup_phase=startup_phase, + error=error, + force_exit=force_exit, + wallet_configured=snapshot.wallet_configured, + new_install=not snapshot.reconciled, + reconciled=snapshot.reconciled, + ) + + +def _distinct_automation_ids_from_states( + automation_states: list[protocol_models.AutomationState], +) -> list[str]: + return sorted({ + automation_state.id + for automation_state in automation_states + if automation_state.id + }) + + +async def complete_reconcile_automations( + config: typing.Optional[configuration.Configuration] = None, +) -> None: + global _reconcile_completed + if not journal_module.is_journal_enabled(): + return + if _reconcile_completed: + return + resolved_config = config if config is not None else journal_state.get_config() + if resolved_config is not None: + journal_module.initialize_for_config(resolved_config) + snapshot = build_existing_config_snapshot() + if not snapshot.reconciled: + return + # Lazy import: octobot_node.scheduler pulls octobot_flow → octobot.community (circular at import time). + import octobot_node.scheduler as scheduler_module + import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader + + if not scheduler_module.is_initialized(): + return + + automation_ids: set[str] = set() + running_automation_count = 0 + for wallet_id in collection_providers.AccountProvider.instance().list_collectable_wallet_ids(): + automation_states = await automation_states_loader.load_protocol_automation_states( + wallet_id, + statuses=None, + ) + automation_ids.update(_distinct_automation_ids_from_states(automation_states)) + running_automation_count += sum( + 1 + for automation_state in automation_states + if automation_state.status == protocol_models.WorkflowStatus.RUNNING + ) + journal_recording.record_reconcile_completed( + automation_count=len(automation_ids), + running_automation_count=running_automation_count, + ) + _reconcile_completed = True diff --git a/octobot/community/node_journal/models.py b/octobot/community/node_journal/models.py new file mode 100644 index 0000000000..90d155275f --- /dev/null +++ b/octobot/community/node_journal/models.py @@ -0,0 +1,211 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses + +import octobot_commons.dataclasses as commons_dataclasses + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events + + +@dataclasses.dataclass +class _NodeJournalMinimizableDataclass(commons_dataclasses.MinimizableDataclass): + def to_dict(self, include_default_values: bool = False) -> dict: + return super().to_dict(include_default_values=include_default_values) + + +@dataclasses.dataclass +class JournalEventAttributes(_NodeJournalMinimizableDataclass): + account_count: int | None = None + account_id: str | None = None + account_ids: list[str] | None = None + account_state: str | None = None + action_type: str | None = None + automation_count: int | None = None + automation_id: str | None = None + running_automation_count: int | None = None + backend: str | None = None + cancel_orders: bool | None = None + collection: str | None = None + configured: bool | None = None + configuration_type: str | None = None + connection_sequence: int | None = None + duration_since_install_start: float | None = None + error_category: str | None = None + error_message: str | None = None + error_origin: str | None = None + error_status: str | None = None + exchange_name: str | None = None + failure_reason: str | None = None + flow_subtype: str | None = None + force_exit: bool | None = None + http_status: int | None = None + init_phase: str | None = None + is_reconnect: bool | None = None + is_simulated: bool | None = None + new_install: bool | None = None + node_type: str | None = None + octobot_kind: str | None = None + operation: str | None = None + prior_gap_seconds: float | None = None + provider: str | None = None + reconciled: bool | None = None + recovery_action: str | None = None + refreshed_count: int | None = None + retriable: bool | None = None + raw_event_name: str | None = None + setup_method: str | None = None + source: str | None = None + startup_phase: str | None = None + strategy_id: str | None = None + sync_session_id: str | None = None + user_action_id: str | None = None + wallet_configured: bool | None = None + + @classmethod + def merge(cls, base: "JournalEventAttributes | None", overrides: dict | None) -> "JournalEventAttributes": + merged = dataclasses.asdict(base) if base is not None else {} + if overrides: + known_fields = cls.get_field_names() + for raw_key, value in overrides.items(): + if raw_key in known_fields: + merged[raw_key] = value + return cls(**merged) + + +@dataclasses.dataclass +class JournalEventLine(_NodeJournalMinimizableDataclass): + event: journal_events.NodeJournalEvent + timestamp: float + session_id: str + install_id: str + app_version: str + distribution: str + onboarding_complete: bool + attributes: JournalEventAttributes + recorded: bool = True + + def to_dict(self, include_default_values: bool = False) -> dict: + serialized = super().to_dict(include_default_values=include_default_values) + event_line_field = journal_enums.JournalEventLineField + serialized[event_line_field.EVENT.value] = self.event.value + if self.attributes is not None: + serialized[event_line_field.ATTRIBUTES.value] = self.attributes.to_dict( + include_default_values=include_default_values, + ) + else: + serialized[event_line_field.ATTRIBUTES.value] = {} + if not self.recorded: + serialized[event_line_field.RECORDED.value] = False + elif not include_default_values and serialized.get(event_line_field.RECORDED.value) is True: + serialized.pop(event_line_field.RECORDED.value, None) + return serialized + + def to_storage_dict(self) -> dict: + event_line_field = journal_enums.JournalEventLineField + serialized = { + event_line_field.EVENT.value: self.event.value, + event_line_field.TIMESTAMP.value: self.timestamp, + } + if self.attributes is not None: + attributes_dict = self.attributes.to_dict() + if attributes_dict: + serialized[event_line_field.ATTRIBUTES.value] = attributes_dict + if not self.recorded: + serialized[event_line_field.RECORDED.value] = False + return serialized + + @classmethod + def from_dict(cls, event_line: dict) -> "JournalEventLine": + event_line_field = journal_enums.JournalEventLineField + parsed_event, coerced_raw_event_name = journal_events.coerce_node_journal_event( + event_line[event_line_field.EVENT.value], + ) + attributes = JournalEventAttributes.from_dict(event_line.get(event_line_field.ATTRIBUTES.value)) + if attributes is None: + attributes = JournalEventAttributes() + if coerced_raw_event_name is not None: + attributes = JournalEventAttributes.merge( + attributes, + {"raw_event_name": coerced_raw_event_name}, + ) + return cls( + event=parsed_event, + timestamp=float(event_line[event_line_field.TIMESTAMP.value]), + session_id=str(event_line.get(event_line_field.SESSION_ID.value, "")), + install_id=str(event_line.get(event_line_field.INSTALL_ID.value, "")), + app_version=str(event_line.get(event_line_field.APP_VERSION.value, "")), + distribution=str(event_line.get(event_line_field.DISTRIBUTION.value, "")), + onboarding_complete=bool(event_line.get(event_line_field.ONBOARDING_COMPLETE.value, False)), + attributes=attributes, + recorded=event_line.get(event_line_field.RECORDED.value, True), + ) + + +@dataclasses.dataclass +class FirstFailureInfo(_NodeJournalMinimizableDataclass): + event: str + timestamp: float | None + error_category: str | None + error_message: str | None + + @classmethod + def from_dict(cls, payload: dict | None) -> "FirstFailureInfo | None": + if payload is None: + return None + return super().from_dict(payload) + + +@dataclasses.dataclass +class JourneySummary(_NodeJournalMinimizableDataclass): + onboarding_complete: bool + furthest_step_reached: str | None + last_successful_step: str | None + first_failure: FirstFailureInfo | None + retry_counts: dict[str, int] + step_durations_seconds: dict[str, float] + step_deltas_seconds: dict[str, float] + external_connect_count: int + first_external_connect_at: float | None + last_external_connect_at: float | None + longest_connect_gap_seconds: float | None + ui_blocking_issues_count: int = 0 + + +@dataclasses.dataclass +class UploadEnvelope(_NodeJournalMinimizableDataclass): + install_id: str + app_version: str + onboarding_started_at: float | None + onboarding_complete: bool + journey_summary: JourneySummary + events: list[JournalEventLine] + uploaded: bool + ready: bool + event_count: int + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + def to_dict(self, include_default_values: bool = False) -> dict: + serialized = super().to_dict(include_default_values=include_default_values) + serialized["events"] = [event_line.to_storage_dict() for event_line in self.events] + serialized["journey_summary"] = self.journey_summary.to_dict( + include_default_values=include_default_values, + ) + return serialized diff --git a/octobot/community/node_journal/recording/__init__.py b/octobot/community/node_journal/recording/__init__.py new file mode 100644 index 0000000000..c51123004f --- /dev/null +++ b/octobot/community/node_journal/recording/__init__.py @@ -0,0 +1,99 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.recording.accounts as accounts_module +import octobot.community.node_journal.recording.automations as automations_module +import octobot.community.node_journal.recording.process as process_module +import octobot.community.node_journal.recording.strategies as strategies_module +import octobot.community.node_journal.recording.sync as sync_module +import octobot.community.node_journal.recording.user_actions as user_actions_module +import octobot.community.node_journal.recording.wallet as wallet_module + +record_account_auth_create_succeeded = accounts_module.record_account_auth_create_succeeded +record_account_auth_deleted = accounts_module.record_account_auth_deleted +record_account_deleted = accounts_module.record_account_deleted +record_account_edit_succeeded = accounts_module.record_account_edit_succeeded +record_account_validated = accounts_module.record_account_validated +record_account_validated_from_account = accounts_module.record_account_validated_from_account +record_account_validation_failed = accounts_module.record_account_validation_failed +record_accounts_refreshed = accounts_module.record_accounts_refreshed + +record_automation_restarted = automations_module.record_automation_restarted +record_automation_run_errored = automations_module.record_automation_run_errored +record_automation_started = automations_module.record_automation_started +record_automation_stopped = automations_module.record_automation_stopped +record_first_automation_started = automations_module.record_first_automation_started +record_new_automation_created = automations_module.record_new_automation_created +record_new_automation_created_from_strategy = automations_module.record_new_automation_created_from_strategy + +record_existing_config_detected = process_module.record_existing_config_detected +record_process_startup_failed = process_module.record_process_startup_failed +record_process_startup_succeeded = process_module.record_process_startup_succeeded +record_reconcile_completed = process_module.record_reconcile_completed +record_scheduler_init_failed = process_module.record_scheduler_init_failed + +record_strategy_create_succeeded = strategies_module.record_strategy_create_succeeded +record_strategy_edit_succeeded = strategies_module.record_strategy_edit_succeeded + +on_user_data_pull_succeeded = sync_module.on_user_data_pull_succeeded +record_sync_read_failed = sync_module.record_sync_read_failed +record_sync_storage_event = sync_module.record_sync_storage_event +reset_sync_tracker_after_node_startup = sync_module.reset_sync_tracker_after_node_startup + +record_executor_failure = user_actions_module.record_executor_failure +record_external_action_failed = user_actions_module.record_external_action_failed +record_external_action_received = user_actions_module.record_external_action_received + +record_wallet_operation_failed = wallet_module.record_wallet_operation_failed +record_wallet_setup_attempt = wallet_module.record_wallet_setup_attempt +record_wallet_setup_failed = wallet_module.record_wallet_setup_failed +record_wallet_setup_succeeded = wallet_module.record_wallet_setup_succeeded + +__all__ = [ + "on_user_data_pull_succeeded", + "record_account_auth_create_succeeded", + "record_account_auth_deleted", + "record_account_deleted", + "record_account_edit_succeeded", + "record_account_validated", + "record_account_validated_from_account", + "record_account_validation_failed", + "record_accounts_refreshed", + "record_automation_restarted", + "record_automation_run_errored", + "record_automation_started", + "record_automation_stopped", + "record_executor_failure", + "record_existing_config_detected", + "record_external_action_failed", + "record_external_action_received", + "record_first_automation_started", + "record_new_automation_created", + "record_new_automation_created_from_strategy", + "record_process_startup_failed", + "record_process_startup_succeeded", + "record_reconcile_completed", + "record_scheduler_init_failed", + "record_strategy_create_succeeded", + "record_strategy_edit_succeeded", + "record_sync_read_failed", + "record_sync_storage_event", + "record_wallet_operation_failed", + "record_wallet_setup_attempt", + "record_wallet_setup_failed", + "record_wallet_setup_succeeded", + "reset_sync_tracker_after_node_startup", +] diff --git a/octobot/community/node_journal/recording/_utils.py b/octobot/community/node_journal/recording/_utils.py new file mode 100644 index 0000000000..11187f0df8 --- /dev/null +++ b/octobot/community/node_journal/recording/_utils.py @@ -0,0 +1,21 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import enum + + +def enum_value(value: enum.StrEnum) -> str: + return value.value if isinstance(value, enum.StrEnum) else str(value) diff --git a/octobot/community/node_journal/recording/accounts.py b/octobot/community/node_journal/recording/accounts.py new file mode 100644 index 0000000000..72f5fc2e56 --- /dev/null +++ b/octobot/community/node_journal/recording/accounts.py @@ -0,0 +1,145 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording.classify as journal_classify + + +def record_account_validated_from_account( + checked_account: protocol_models.Account, + user_id: str, + *, + user_action_id: str | None = None, +) -> None: + if checked_account.state is None or checked_account.state.status != protocol_models.AccountStatus.VALID: + return + exchange_name = journal_classify.resolve_account_exchange_name(checked_account, user_id) + record_account_validated( + account_id=checked_account.id or "", + is_simulated=checked_account.is_simulated, + exchange_name=exchange_name or "", + user_action_id=user_action_id, + ) + + +def record_account_validated( + *, + account_id: str = "", + is_simulated: bool, + exchange_name: str | None, + user_action_id: str | None = None, + account_count: int | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + account_id=account_id or None, + is_simulated=is_simulated, + exchange_name=exchange_name, + user_action_id=user_action_id, + account_count=account_count, + ) + journal_module.record(journal_events.NodeJournalEvent.ACCOUNT_VALIDATED, attributes=attributes) + + +def record_account_validation_failed( + *, + is_simulated: bool, + exchange_name: str | None, + error: BaseException, + user_action_id: str | None = None, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.ACCOUNT_VALIDATION_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + is_simulated=is_simulated, + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_account_auth_create_succeeded( + *, + exchange_name: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_account_edit_succeeded( + *, + account_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_EDIT_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + account_id=account_id, + user_action_id=user_action_id, + ), + ) + + +def record_account_deleted( + *, + account_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_DELETED, + attributes=journal_models.JournalEventAttributes( + account_id=account_id, + user_action_id=user_action_id, + ), + ) + + +def record_account_auth_deleted( + *, + exchange_name: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_DELETED, + attributes=journal_models.JournalEventAttributes( + exchange_name=exchange_name, + user_action_id=user_action_id, + ), + ) + + +def record_accounts_refreshed( + *, + account_ids: list[str], + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.ACCOUNTS_REFRESHED, + attributes=journal_models.JournalEventAttributes( + account_ids=list(account_ids), + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/automations.py b/octobot/community/node_journal/recording/automations.py new file mode 100644 index 0000000000..c42f92008f --- /dev/null +++ b/octobot/community/node_journal/recording/automations.py @@ -0,0 +1,162 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.classify as journal_classify +import octobot.community.node_journal.state as journal_state + + +def record_new_automation_created_from_strategy( + automation_id: str, + strategy: protocol_models.Strategy, + *, + user_action_id: str | None = None, + source: str | None = None, +) -> None: + persisted_state = journal_state.load_persisted_state() + if automation_id in persisted_state.tracked_automation_ids: + return + octobot_kind, flow_subtype = journal_classify.classify_octobot_kind(strategy) + is_first_automation = persisted_state.first_automation_started_at is None + persisted_state.tracked_automation_ids.append(automation_id) + journal_state.save_persisted_state(persisted_state) + record_new_automation_created( + automation_id=automation_id, + octobot_kind=octobot_kind.value, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + is_first_automation=is_first_automation, + ) + + +def record_first_automation_started( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.FIRST_AUTOMATION_STARTED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + octobot_kind=recording_utils.enum_value(octobot_kind), + flow_subtype=flow_subtype, + user_action_id=user_action_id, + duration_since_install_start=journal_state.duration_since_install_start(), + ), + ) + + +def record_automation_started( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, + source: str | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + automation_id=automation_id, + octobot_kind=recording_utils.enum_value(octobot_kind), + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + ) + journal_module.record(journal_events.NodeJournalEvent.AUTOMATION_STARTED, attributes=attributes) + + +def record_new_automation_created( + *, + automation_id: str, + octobot_kind: journal_enums.OctobotKind, + flow_subtype: str | None, + user_action_id: str | None = None, + source: str | None = None, + is_first_automation: bool, +) -> None: + if is_first_automation: + record_first_automation_started( + automation_id=automation_id, + octobot_kind=octobot_kind, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + ) + return + record_automation_started( + automation_id=automation_id, + octobot_kind=octobot_kind, + flow_subtype=flow_subtype, + user_action_id=user_action_id, + source=source, + ) + + +def record_automation_run_errored( + *, + automation_id: str, + error_status: str, + error_origin: str, + error: BaseException, + retriable: bool, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.AUTOMATION_RUN_ERRORED, + error=error, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + error_status=error_status, + error_origin=error_origin, + retriable=retriable, + ), + ) + + +def record_automation_stopped( + *, + automation_id: str, + cancel_orders: bool, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.AUTOMATION_STOPPED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + cancel_orders=cancel_orders, + user_action_id=user_action_id, + ), + ) + + +def record_automation_restarted( + *, + automation_id: str, + user_action_id: str | None = None, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.AUTOMATION_RESTARTED, + attributes=journal_models.JournalEventAttributes( + automation_id=automation_id, + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/classify.py b/octobot/community/node_journal/recording/classify.py new file mode 100644 index 0000000000..6d6652bf0b --- /dev/null +++ b/octobot/community/node_journal/recording/classify.py @@ -0,0 +1,97 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot.constants as constants +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal.enums as journal_enums + + +def classify_octobot_kind( + strategy: protocol_models.Strategy, +) -> tuple[journal_enums.OctobotKind, str | None]: + configuration_wrapper = strategy.configuration + if configuration_wrapper is None or configuration_wrapper.actual_instance is None: + return journal_enums.OctobotKind.MANUAL, None + inner_configuration = configuration_wrapper.actual_instance + if isinstance(inner_configuration, protocol_models.GenericProcessConfiguration): + return journal_enums.OctobotKind.MANUAL, None + if isinstance(inner_configuration, protocol_models.MarketMakingConfiguration): + return journal_enums.OctobotKind.MARKET_MAKING, None + if isinstance(inner_configuration, protocol_models.CopyConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.COPY.value + if isinstance(inner_configuration, protocol_models.SignalBotConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.SIGNAL_BOT.value + if isinstance(inner_configuration, protocol_models.GenericWorkflowConfiguration): + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.AI_AGENTS.value + if isinstance(inner_configuration, protocol_models.TradingTentaclesConfiguration): + tentacle_name = inner_configuration.name or "" + if not tentacle_name: + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.OTHER.value + return journal_enums.OctobotKind.FLOW, tentacle_name + return journal_enums.OctobotKind.FLOW, journal_enums.FlowSubtype.OTHER.value + + +def configuration_type_from_strategy(strategy: protocol_models.Strategy) -> journal_enums.ConfigurationType: + configuration_wrapper = strategy.configuration + if configuration_wrapper is None or configuration_wrapper.actual_instance is None: + return journal_enums.ConfigurationType.GENERIC_PROCESS + inner_configuration = configuration_wrapper.actual_instance + if isinstance(inner_configuration, protocol_models.GenericProcessConfiguration): + return journal_enums.ConfigurationType.GENERIC_PROCESS + if isinstance(inner_configuration, protocol_models.MarketMakingConfiguration): + return journal_enums.ConfigurationType.MARKET_MAKING + if isinstance(inner_configuration, protocol_models.CopyConfiguration): + return journal_enums.ConfigurationType.COPY + if isinstance(inner_configuration, protocol_models.SignalBotConfiguration): + return journal_enums.ConfigurationType.SIGNAL_BOT + if isinstance(inner_configuration, protocol_models.GenericWorkflowConfiguration): + return journal_enums.ConfigurationType.GENERIC_WORKFLOW + if isinstance(inner_configuration, protocol_models.TradingTentaclesConfiguration): + return journal_enums.ConfigurationType.TRADING_TENTACLES + return journal_enums.ConfigurationType.GENERIC_PROCESS + + +def resolve_account_exchange_name( + checked_account: protocol_models.Account, + user_id: str, +) -> str: + specifics = checked_account.specifics + if specifics is None or specifics.actual_instance is None: + return constants.METRICS_GENERIC_EXCHANGE_NAME + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return constants.METRICS_GENERIC_EXCHANGE_NAME + config_ids = exchange_account.exchange_config_ids or [] + if not config_ids: + return constants.METRICS_GENERIC_EXCHANGE_NAME + import octobot_sync.sync.collection_providers as collection_providers + try: + exchange_config = collection_providers.AccountProvider.instance().get_exchange_config( + user_id, + config_ids[0], + ) + except collection_errors.CollectionNoDataError: + return constants.METRICS_GENERIC_EXCHANGE_NAME + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + protocol_models.TradingType.SPOT + ) + if exchange_type is None: + return exchange_config.exchange + return exchange_config.exchange diff --git a/octobot/community/node_journal/recording/process.py b/octobot/community/node_journal/recording/process.py new file mode 100644 index 0000000000..66bebe8da7 --- /dev/null +++ b/octobot/community/node_journal/recording/process.py @@ -0,0 +1,111 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.sync as sync_module +import octobot.community.node_journal.state as journal_state + + +def record_process_startup_succeeded( + *, + wallet_configured: bool, + new_install: bool, + reconciled: bool, +) -> None: + sync_module.reset_sync_tracker_after_node_startup() + journal_module.record( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + wallet_configured=wallet_configured, + new_install=new_install, + reconciled=reconciled, + duration_since_install_start=journal_state.duration_since_install_start(), + ), + ) + + +def record_process_startup_failed( + *, + startup_phase: journal_enums.JournalStartupPhase, + error: BaseException, + force_exit: bool, + wallet_configured: bool | None = None, + new_install: bool | None = None, + reconciled: bool | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + startup_phase=recording_utils.enum_value(startup_phase), + force_exit=force_exit, + ) + if wallet_configured is not None: + attributes.wallet_configured = wallet_configured + if new_install is not None: + attributes.new_install = new_install + if reconciled is not None: + attributes.reconciled = reconciled + journal_module.record_failure( + journal_events.NodeJournalEvent.NODE_PROCESS_STARTUP_FAILED, + error=error, + attributes=attributes, + ) + + +def record_existing_config_detected( + *, + wallet_configured: bool, + account_count: int, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.EXISTING_CONFIG_DETECTED, + attributes=journal_models.JournalEventAttributes( + wallet_configured=wallet_configured, + account_count=account_count, + ), + ) + + +def record_reconcile_completed( + *, + automation_count: int, + running_automation_count: int, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.RECONCILE_COMPLETED, + attributes=journal_models.JournalEventAttributes( + automation_count=automation_count, + running_automation_count=running_automation_count, + ), + ) + + +def record_scheduler_init_failed( + *, + init_phase: journal_enums.JournalInitPhase, + backend: journal_enums.JournalSchedulerBackend, + error: BaseException, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.SCHEDULER_INIT_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + init_phase=recording_utils.enum_value(init_phase), + backend=recording_utils.enum_value(backend), + ), + ) diff --git a/octobot/community/node_journal/recording/strategies.py b/octobot/community/node_journal/recording/strategies.py new file mode 100644 index 0000000000..3857c251c3 --- /dev/null +++ b/octobot/community/node_journal/recording/strategies.py @@ -0,0 +1,64 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.recording.classify as journal_classify + + +def record_strategy_create_succeeded( + *, + strategy_id: str, + strategy: protocol_models.Strategy | None = None, + configuration_type: journal_enums.ConfigurationType | str | None = None, + user_action_id: str | None = None, +) -> None: + resolved_configuration_type = configuration_type + if strategy is not None: + resolved_configuration_type = journal_classify.configuration_type_from_strategy(strategy) + journal_module.record( + journal_events.NodeJournalEvent.STRATEGY_CREATE_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + strategy_id=strategy_id, + configuration_type=recording_utils.enum_value(resolved_configuration_type), + user_action_id=user_action_id, + ), + ) + + +def record_strategy_edit_succeeded( + *, + strategy_id: str, + strategy: protocol_models.Strategy | None = None, + configuration_type: journal_enums.ConfigurationType | str | None = None, + user_action_id: str | None = None, +) -> None: + resolved_configuration_type = configuration_type + if strategy is not None: + resolved_configuration_type = journal_classify.configuration_type_from_strategy(strategy) + journal_module.record( + journal_events.NodeJournalEvent.STRATEGY_EDIT_SUCCEEDED, + attributes=journal_models.JournalEventAttributes( + strategy_id=strategy_id, + configuration_type=recording_utils.enum_value(resolved_configuration_type), + user_action_id=user_action_id, + ), + ) diff --git a/octobot/community/node_journal/recording/sync.py b/octobot/community/node_journal/recording/sync.py new file mode 100644 index 0000000000..0b20eed569 --- /dev/null +++ b/octobot/community/node_journal/recording/sync.py @@ -0,0 +1,107 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import time +import uuid + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils +import octobot.community.node_journal.state as journal_state + + +_tracker_reset_after_startup = False + + +def reset_sync_tracker_after_node_startup() -> None: + global _tracker_reset_after_startup + _tracker_reset_after_startup = True + + +def on_user_data_pull_succeeded(*, sync_user_id: str, collection: str) -> None: + global _tracker_reset_after_startup + if collection != "user-data": + return + now = time.time() + persisted_state = journal_state.load_persisted_state() + is_first_ever = persisted_state.last_user_data_pull_at is None + prior_gap_seconds = None + is_reconnect = False + if _tracker_reset_after_startup and not is_first_ever: + is_reconnect = True + prior_gap_seconds = round(now - persisted_state.last_user_data_pull_at, 3) + elif persisted_state.last_user_data_pull_at is not None: + gap_seconds = now - persisted_state.last_user_data_pull_at + if gap_seconds > journal_constants.SYNC_SESSION_GAP_SECONDS: + is_reconnect = True + prior_gap_seconds = round(gap_seconds, 3) + if is_first_ever or is_reconnect or _tracker_reset_after_startup: + persisted_state.connection_sequence += 1 + attributes = journal_models.JournalEventAttributes( + collection=collection, + sync_session_id=str(uuid.uuid4()), + connection_sequence=persisted_state.connection_sequence, + is_reconnect=not is_first_ever, + duration_since_install_start=journal_state.duration_since_install_start(now), + prior_gap_seconds=prior_gap_seconds, + ) + journal_module.record( + journal_events.NodeJournalEvent.EXTERNAL_INTERFACE_CONNECTED, + attributes=attributes, + ) + persisted_state.last_user_data_pull_at = now + journal_state.save_persisted_state(persisted_state) + _tracker_reset_after_startup = False + + +def record_sync_read_failed( + *, + collection: str, + failure_reason: journal_enums.SyncReadFailureReason, + error: BaseException | None = None, + error_message: str | None = None, +) -> None: + journal_module.record_failure( + journal_events.NodeJournalEvent.SYNC_READ_FAILED, + error=error, + error_message=error_message, + attributes=journal_models.JournalEventAttributes( + collection=collection, + failure_reason=recording_utils.enum_value(failure_reason), + ), + ) + + +def record_sync_storage_event( + event: journal_events.NodeJournalEvent, + *, + collection: str, + provider: journal_enums.SyncStorageProvider, + error: BaseException | None = None, + recovery_action: journal_enums.SyncStorageRecoveryAction | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + collection=collection, + provider=recording_utils.enum_value(provider), + recovery_action=recording_utils.enum_value(recovery_action) if recovery_action is not None else None, + ) + if error is None: + journal_module.record(event, attributes=attributes) + return + journal_module.record_failure(event, error=error, attributes=attributes) diff --git a/octobot/community/node_journal/recording/user_actions.py b/octobot/community/node_journal/recording/user_actions.py new file mode 100644 index 0000000000..ad32d7dcad --- /dev/null +++ b/octobot/community/node_journal/recording/user_actions.py @@ -0,0 +1,152 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording.accounts as accounts_module + + +ACTION_EVENT_MAP: dict[ + protocol_models.UserActionType, + tuple[journal_events.NodeJournalEvent, journal_events.NodeJournalEvent | None], +] = { + protocol_models.UserActionType.ACCOUNT_AUTH_CREATE: ( + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.ACCOUNT_AUTH_CREATE_FAILED, + ), + protocol_models.UserActionType.ACCOUNT_CREATE: ( + journal_events.NodeJournalEvent.ACCOUNT_CREATE_ATTEMPT, + None, + ), + protocol_models.UserActionType.ACCOUNT_EDIT: ( + journal_events.NodeJournalEvent.ACCOUNT_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.ACCOUNT_EDIT_FAILED, + ), + protocol_models.UserActionType.STRATEGY_CREATE: ( + journal_events.NodeJournalEvent.STRATEGY_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.STRATEGY_CREATE_FAILED, + ), + protocol_models.UserActionType.STRATEGY_EDIT: ( + journal_events.NodeJournalEvent.STRATEGY_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.STRATEGY_EDIT_FAILED, + ), + protocol_models.UserActionType.AUTOMATION_CREATE: ( + journal_events.NodeJournalEvent.AUTOMATION_CREATE_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_CREATE_FAILED, + ), + protocol_models.UserActionType.AUTOMATION_EDIT: ( + journal_events.NodeJournalEvent.AUTOMATION_EDIT_ATTEMPT, + journal_events.NodeJournalEvent.AUTOMATION_EDIT_FAILED, + ), +} + + +def record_external_action_received( + user_action: protocol_models.UserAction, + *, + source: str, +) -> None: + action_type = _resolve_user_action_type(user_action) + journal_module.record( + journal_events.NodeJournalEvent.EXTERNAL_ACTION_RECEIVED, + attributes=journal_models.JournalEventAttributes( + source=source, + action_type=action_type.value if action_type is not None else "unknown", + user_action_id=user_action.id, + ), + ) + _record_action_attempt(user_action, source=source) + + +def record_external_action_failed( + user_action: protocol_models.UserAction, + *, + source: str, + error: BaseException, +) -> None: + action_type = _resolve_user_action_type(user_action) + journal_module.record_failure( + journal_events.NodeJournalEvent.EXTERNAL_ACTION_FAILED, + error=error, + attributes=journal_models.JournalEventAttributes( + source=source, + action_type=action_type.value if action_type is not None else "unknown", + user_action_id=user_action.id, + ), + ) + _record_typed_action_failure(user_action, error=error) + + +def record_executor_failure( + user_action: protocol_models.UserAction, + *, + error: BaseException, +) -> None: + _record_typed_action_failure(user_action, error=error) + + +def _record_action_attempt(user_action: protocol_models.UserAction, *, source: str) -> None: + action_type = _resolve_user_action_type(user_action) + if action_type is None: + return + attempt_event, _failure_event = ACTION_EVENT_MAP.get(action_type, (None, None)) + if attempt_event is None: + return + journal_module.record( + attempt_event, + attributes=journal_models.JournalEventAttributes( + user_action_id=user_action.id, + source=source, + ), + ) + + +def _record_typed_action_failure( + user_action: protocol_models.UserAction, + *, + error: BaseException, +) -> None: + action_type = _resolve_user_action_type(user_action) + if action_type is None: + return + _attempt_event, failure_event = ACTION_EVENT_MAP.get(action_type, (None, None)) + if action_type == protocol_models.UserActionType.ACCOUNT_CREATE: + accounts_module.record_account_validation_failed( + is_simulated=False, + exchange_name=None, + error=error, + user_action_id=user_action.id, + ) + return + if failure_event is None: + return + journal_module.record_failure( + failure_event, + error=error, + attributes=journal_models.JournalEventAttributes(user_action_id=user_action.id), + ) + + +def _resolve_user_action_type( + user_action: protocol_models.UserAction, +) -> protocol_models.UserActionType | None: + configuration = user_action.configuration + if configuration is None or configuration.actual_instance is None: + return None + return configuration.actual_instance.action_type diff --git a/octobot/community/node_journal/recording/wallet.py b/octobot/community/node_journal/recording/wallet.py new file mode 100644 index 0000000000..1889df8cee --- /dev/null +++ b/octobot/community/node_journal/recording/wallet.py @@ -0,0 +1,80 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.recording._utils as recording_utils + + +def record_wallet_setup_attempt( + *, + node_type: str, + setup_method: journal_enums.WalletSetupMethod, +) -> None: + journal_module.record( + journal_events.NodeJournalEvent.WALLET_SETUP_ATTEMPT, + attributes=journal_models.JournalEventAttributes( + node_type=node_type, + setup_method=recording_utils.enum_value(setup_method), + ), + ) + + +def record_wallet_setup_succeeded() -> None: + journal_module.record( + journal_events.NodeJournalEvent.WALLET_SETUP_SUCCEEDED, + attributes=journal_models.JournalEventAttributes(configured=True), + ) + + +def record_wallet_setup_failed( + *, + http_status: int, + failure_reason: journal_enums.WalletSetupFailureReason, + error: BaseException | None = None, + error_message: str | None = None, + setup_method: journal_enums.WalletSetupMethod | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes( + http_status=http_status, + failure_reason=recording_utils.enum_value(failure_reason), + ) + if setup_method is not None: + attributes.setup_method = recording_utils.enum_value(setup_method) + journal_module.record_failure( + journal_events.NodeJournalEvent.WALLET_SETUP_FAILED, + error=error, + error_message=error_message, + attributes=attributes, + ) + + +def record_wallet_operation_failed( + *, + operation: journal_enums.WalletOperation, + error: BaseException, + http_status: int | None = None, +) -> None: + attributes = journal_models.JournalEventAttributes(operation=recording_utils.enum_value(operation)) + if http_status is not None: + attributes.http_status = http_status + journal_module.record_failure( + journal_events.NodeJournalEvent.WALLET_OPERATION_FAILED, + error=error, + attributes=attributes, + ) diff --git a/octobot/community/node_journal/recording_context.py b/octobot/community/node_journal/recording_context.py new file mode 100644 index 0000000000..87d64561dc --- /dev/null +++ b/octobot/community/node_journal/recording_context.py @@ -0,0 +1,132 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import contextlib +import typing + +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.recording as journal_recording + + +class scheduler_init_phase(contextlib.AbstractContextManager): + def __init__( + self, + *, + init_phase: journal_enums.JournalInitPhase, + backend: journal_enums.JournalSchedulerBackend, + on_failure: typing.Callable[..., None], + ) -> None: + self._init_phase = init_phase + self._backend = backend + self._on_failure = on_failure + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + self._on_failure( + init_phase=self._init_phase, + backend=self._backend, + error=exc, + ) + return False + + +class sync_read_operation(contextlib.AbstractContextManager): + def __init__( + self, + *, + resolve_collection: typing.Callable[[], str], + resolve_failure_reason: typing.Callable[[BaseException], str], + ) -> None: + self._resolve_collection = resolve_collection + self._resolve_failure_reason = resolve_failure_reason + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + journal_recording.record_sync_read_failed( + collection=self._resolve_collection(), + failure_reason=self._resolve_failure_reason(exc), + error=exc, + ) + return False + + +class sync_storage_error(contextlib.AbstractContextManager): + def __init__( + self, + *, + event: journal_events.NodeJournalEvent, + collection: str, + provider: journal_enums.SyncStorageProvider = journal_enums.SyncStorageProvider.LOCAL, + recovery_action: journal_enums.SyncStorageRecoveryAction | None = None, + ) -> None: + self._event = event + self._collection = collection + self._provider = provider + self._recovery_action = recovery_action + + def __exit__(self, exc_type, exc, traceback) -> bool: + del traceback + if exc is not None and exc_type is not None: + journal_recording.record_sync_storage_event( + self._event, + collection=self._collection, + provider=self._provider, + error=exc, + recovery_action=self._recovery_action, + ) + return False + + +def raise_wallet_setup_http_error( + *, + http_status: int, + failure_reason: journal_enums.WalletSetupFailureReason, + setup_method: journal_enums.WalletSetupMethod, + detail: str, + error: BaseException | None = None, + error_message: str | None = None, +) -> typing.NoReturn: + journal_recording.record_wallet_setup_failed( + http_status=http_status, + failure_reason=failure_reason, + error=error, + error_message=error_message, + setup_method=setup_method, + ) + from fastapi import HTTPException + raise HTTPException(status_code=http_status, detail=detail) from error + + +def node_api_startup_failure( + *, + error: BaseException, + startup_phase: journal_enums.JournalStartupPhase, + force_exit: bool, + config, + scheduler_init_failure_was_recorded: typing.Callable[[], bool], + record_node_startup_failed: typing.Callable[..., None], +) -> None: + if scheduler_init_failure_was_recorded(): + return + record_node_startup_failed( + error, + startup_phase=startup_phase, + force_exit=force_exit, + config=config, + ) \ No newline at end of file diff --git a/octobot/community/node_journal/state.py b/octobot/community/node_journal/state.py new file mode 100644 index 0000000000..b9b76e6208 --- /dev/null +++ b/octobot/community/node_journal/state.py @@ -0,0 +1,225 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import logging +import os +import time +import typing +import uuid + +import octobot_commons.configuration as configuration +import octobot_commons.json_util as json_util +import octobot_commons.user_root_folder_provider as user_root_folder_provider + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.journal as journal_module + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class JournalPersistedState: + install_id: str + onboarding_started_at: float | None = None + first_automation_started_at: float | None = None + connection_sequence: int = 0 + last_user_data_pull_at: float | None = None + onboarding_complete: bool = False + tracked_automation_ids: list[str] = dataclasses.field(default_factory=list) + + +_session_id: str | None = None +_persisted_state: JournalPersistedState | None = None +_config: configuration.Configuration | None = None + + +def _get_journal_section(config: configuration.Configuration) -> dict: + journal_section = config.config.setdefault(journal_constants.CONFIG_JOURNAL_SECTION, {}) + if not isinstance(journal_section, dict): + logger.error( + "%s must be a mapping in config", + journal_constants.CONFIG_JOURNAL_SECTION, + ) + return {} + return journal_section + + +def bind_config(config: configuration.Configuration | None) -> None: + global _config, _persisted_state + _config = config + _persisted_state = None + + +def get_config() -> configuration.Configuration | None: + return _config + + +def get_session_id() -> str: + global _session_id + if _session_id is None: + _session_id = str(uuid.uuid4()) + return _session_id + + +def reset_session_id() -> str: + global _session_id + _session_id = str(uuid.uuid4()) + return _session_id + + +def get_journal_directory() -> str: + user_root = user_root_folder_provider.get_user_root_folder() + return os.path.join(user_root, journal_constants.JOURNAL_DIR_NAME) + + +def _is_valid_manifest(manifest: typing.Any) -> bool: + manifest_field = journal_enums.JournalManifestField + if not isinstance(manifest, dict): + return False + if manifest.get(manifest_field.SCHEMA.value) != journal_constants.JOURNAL_SCHEMA_VERSION: + return False + install_id = manifest.get(manifest_field.INSTALL_ID.value) + return isinstance(install_id, str) and bool(install_id) + + +def ensure_journal_manifest(journal_directory: str | None = None) -> dict: + journal_dir = journal_directory or get_journal_directory() + os.makedirs(journal_dir, exist_ok=True) + manifest_path = os.path.join(journal_dir, journal_constants.MANIFEST_FILE_NAME) + manifest = None + if os.path.isfile(manifest_path): + manifest = json_util.read_file(manifest_path, raise_errors=False, on_error_value={}) + if manifest is not None and _is_valid_manifest(manifest): + return manifest + logger.warning("Recreating journal manifest at %s", manifest_path) + persisted_state = load_persisted_state() + manifest_field = journal_enums.JournalManifestField + recreated_manifest = { + manifest_field.SCHEMA.value: journal_constants.JOURNAL_SCHEMA_VERSION, + manifest_field.INSTALL_ID.value: persisted_state.install_id, + } + json_util.safe_dump(recreated_manifest, manifest_path) + return recreated_manifest + + +def load_persisted_state(config: configuration.Configuration | None = None) -> JournalPersistedState: + return journal_module.run_journal_operation( + "load_persisted_state", + lambda: _load_persisted_state(config), + default=_default_persisted_state(), + ) + + +def save_persisted_state(state: JournalPersistedState, config: configuration.Configuration | None = None) -> None: + journal_module.run_journal_operation( + "save_persisted_state", + lambda: _save_persisted_state(state, config), + default=None, + ) + + +def mark_first_automation_started(now: float | None = None) -> None: + journal_module.run_journal_operation( + "mark_first_automation_started", + lambda: _mark_first_automation_started(now), + default=None, + ) + + +def _default_persisted_state() -> JournalPersistedState: + global _persisted_state + if _persisted_state is not None: + return _persisted_state + _persisted_state = JournalPersistedState(install_id=str(uuid.uuid4())) + return _persisted_state + + +def _load_persisted_state(config: configuration.Configuration | None) -> JournalPersistedState: + global _persisted_state + resolved_config = config if config is not None else _config + if resolved_config is None: + return _default_persisted_state() + if _persisted_state is not None and config is None: + return _persisted_state + journal_section = _get_journal_section(resolved_config) + stored_install_id = journal_section.get(journal_constants.CONFIG_INSTALL_ID) + if not stored_install_id: + stored_install_id = str(uuid.uuid4()) + journal_section[journal_constants.CONFIG_INSTALL_ID] = stored_install_id + if journal_section.get(journal_constants.CONFIG_ONBOARDING_STARTED_AT) is None: + journal_section[journal_constants.CONFIG_ONBOARDING_STARTED_AT] = time.time() + resolved_config.save() + _persisted_state = JournalPersistedState( + install_id=str(stored_install_id), + onboarding_started_at=_optional_float(journal_section.get(journal_constants.CONFIG_ONBOARDING_STARTED_AT)), + first_automation_started_at=_optional_float( + journal_section.get(journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT) + ), + connection_sequence=int(journal_section.get(journal_constants.CONFIG_CONNECTION_SEQUENCE, 0) or 0), + last_user_data_pull_at=_optional_float(journal_section.get(journal_constants.CONFIG_LAST_USER_DATA_PULL_AT)), + onboarding_complete=journal_section.get(journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT) is not None, + tracked_automation_ids=list(journal_section.get(journal_constants.CONFIG_TRACKED_AUTOMATION_IDS, []) or []), + ) + return _persisted_state + + +def _save_persisted_state( + state: JournalPersistedState, + config: configuration.Configuration | None = None, +) -> None: + global _persisted_state + resolved_config = config if config is not None else _config + if resolved_config is None: + _persisted_state = state + return + journal_section = _get_journal_section(resolved_config) + journal_section[journal_constants.CONFIG_INSTALL_ID] = state.install_id + if state.onboarding_started_at is not None: + journal_section[journal_constants.CONFIG_ONBOARDING_STARTED_AT] = state.onboarding_started_at + if state.first_automation_started_at is not None: + journal_section[journal_constants.CONFIG_FIRST_AUTOMATION_STARTED_AT] = state.first_automation_started_at + journal_section[journal_constants.CONFIG_CONNECTION_SEQUENCE] = state.connection_sequence + if state.last_user_data_pull_at is not None: + journal_section[journal_constants.CONFIG_LAST_USER_DATA_PULL_AT] = state.last_user_data_pull_at + journal_section[journal_constants.CONFIG_TRACKED_AUTOMATION_IDS] = list(state.tracked_automation_ids) + resolved_config.save() + _persisted_state = state + + +def _mark_first_automation_started(now: float | None) -> None: + state = _load_persisted_state(None) + if state.first_automation_started_at is not None: + return + emit_now = time.time() if now is None else now + state.first_automation_started_at = emit_now + state.onboarding_complete = True + _save_persisted_state(state) + + +def duration_since_install_start(now: float | None = None) -> float | None: + persisted_state = load_persisted_state() + if persisted_state.onboarding_started_at is None: + return None + emit_now = time.time() if now is None else now + return round(max(0.0, emit_now - persisted_state.onboarding_started_at), 3) + + +def _optional_float(value: typing.Any) -> float | None: + if value is None: + return None + return float(value) diff --git a/octobot/community/node_journal/storage_hydration.py b/octobot/community/node_journal/storage_hydration.py new file mode 100644 index 0000000000..221f211549 --- /dev/null +++ b/octobot/community/node_journal/storage_hydration.py @@ -0,0 +1,87 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import logging + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.state as journal_state + +logger = logging.getLogger(__name__) + + +def _derive_onboarding_complete( + event_timestamp: float, + persisted_state: journal_state.JournalPersistedState, +) -> bool: + cutoff = persisted_state.first_automation_started_at + if cutoff is None: + return False + return event_timestamp > cutoff + + +def _log_missing_hydration_field(field_name: str, *, missing_data_warnings: set[str] | None) -> None: + if missing_data_warnings is not None and field_name in missing_data_warnings: + return + logger.warning("Journal event hydration missing %s", field_name) + if missing_data_warnings is not None: + missing_data_warnings.add(field_name) + + +def hydrate_storage_event( + raw: dict, + ctx: dict, + manifest: dict | None, + persisted_state: journal_state.JournalPersistedState, + *, + missing_data_warnings: set[str] | None = None, +) -> journal_models.JournalEventLine: + event_line_field = journal_enums.JournalEventLineField + manifest_field = journal_enums.JournalManifestField + context_field = journal_enums.JournalStorageContextField + + install_id = "" + if isinstance(manifest, dict) and manifest.get(manifest_field.INSTALL_ID.value): + install_id = str(manifest[manifest_field.INSTALL_ID.value]) + else: + _log_missing_hydration_field(manifest_field.INSTALL_ID.value, missing_data_warnings=missing_data_warnings) + + session_id = "" + if ctx.get(context_field.SESSION_ID.value): + session_id = str(ctx[context_field.SESSION_ID.value]) + else: + _log_missing_hydration_field(context_field.SESSION_ID.value, missing_data_warnings=missing_data_warnings) + + app_version = "" + if ctx.get(context_field.APP_VERSION.value): + app_version = str(ctx[context_field.APP_VERSION.value]) + else: + _log_missing_hydration_field(context_field.APP_VERSION.value, missing_data_warnings=missing_data_warnings) + + event_timestamp = float(raw[event_line_field.TIMESTAMP.value]) + expanded_line = { + event_line_field.EVENT.value: raw[event_line_field.EVENT.value], + event_line_field.TIMESTAMP.value: event_timestamp, + event_line_field.SESSION_ID.value: session_id, + event_line_field.INSTALL_ID.value: install_id, + event_line_field.APP_VERSION.value: app_version, + event_line_field.DISTRIBUTION.value: journal_constants.DISTRIBUTION_NODE, + event_line_field.ONBOARDING_COMPLETE.value: _derive_onboarding_complete(event_timestamp, persisted_state), + event_line_field.ATTRIBUTES.value: raw.get(event_line_field.ATTRIBUTES.value), + event_line_field.RECORDED.value: raw.get(event_line_field.RECORDED.value, True), + } + return journal_models.JournalEventLine.from_dict(expanded_line) diff --git a/octobot/community/node_journal/store.py b/octobot/community/node_journal/store.py new file mode 100644 index 0000000000..3d58ba7f4b --- /dev/null +++ b/octobot/community/node_journal/store.py @@ -0,0 +1,321 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import dataclasses +import json +import logging +import os +import threading +import typing + +import octobot.constants as octobot_constants + +import octobot.community.node_journal.constants as journal_constants +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.models as journal_models +import octobot.community.node_journal.storage_hydration as journal_storage_hydration +import octobot.community.node_journal.state as journal_state + +logger = logging.getLogger(__name__) + + +def run_journal_store_operation( + operation_name: str, + operation: typing.Callable[[], typing.Any], + *, + default: typing.Any, +) -> typing.Any: + try: + return operation() + except Exception as exc: + logger.exception("Journal %s failed: %s", operation_name, exc) + return default + + +@dataclasses.dataclass +class _FileWriteState: + last_ctx: dict | None = None + + +class JournalStore: + def __init__( + self, + *, + max_events: int = journal_constants.JOURNAL_MAX_EVENTS, + journal_directory: str | None = None, + ) -> None: + self._max_events = max_events + self._journal_directory = journal_directory or journal_state.get_journal_directory() + self._events_path = os.path.join(self._journal_directory, journal_constants.EVENTS_FILE_NAME) + self._onboarding_path = os.path.join( + self._journal_directory, journal_constants.ONBOARDING_SEGMENT_FILE_NAME + ) + self._lock = threading.Lock() + self._file_write_states: dict[str, _FileWriteState] = {} + os.makedirs(self._journal_directory, exist_ok=True) + journal_state.ensure_journal_manifest(self._journal_directory) + + def append(self, event_line: journal_models.JournalEventLine, *, is_onboarding_segment: bool) -> None: + run_journal_store_operation( + "store.append", + lambda: self._append(event_line, is_onboarding_segment=is_onboarding_segment), + default=None, + ) + + def read_all_events(self) -> list[journal_models.JournalEventLine]: + return run_journal_store_operation( + "store.read_all_events", + self._read_all_events, + default=[], + ) + + def _append(self, event_line: journal_models.JournalEventLine, *, is_onboarding_segment: bool) -> None: + journal_state.ensure_journal_manifest(self._journal_directory) + with self._lock: + if is_onboarding_segment: + self._append_to_file(self._onboarding_path, event_line) + else: + self._append_to_file(self._events_path, event_line) + self._enforce_cap() + + def _read_all_events(self) -> list[journal_models.JournalEventLine]: + manifest = journal_state.ensure_journal_manifest(self._journal_directory) + with self._lock: + pinned_onboarding = self._parse_jsonl_file(self._onboarding_path, manifest=manifest) + main_events = self._parse_jsonl_file(self._events_path, manifest=manifest) + if not pinned_onboarding: + return main_events + merged_by_key = {} + for event_line in pinned_onboarding + main_events: + merged_by_key[self._event_dedup_key(event_line)] = event_line + return sorted(merged_by_key.values(), key=lambda line: line.timestamp) + + def _enforce_cap(self) -> None: + manifest = journal_state.ensure_journal_manifest(self._journal_directory) + main_events = self._parse_jsonl_file(self._events_path, manifest=manifest) + if len(main_events) <= self._max_events: + return + persisted_state = journal_state.load_persisted_state() + onboarding_cutoff = persisted_state.first_automation_started_at + protected_events = [] + evictable_events = [] + for event_line in main_events: + if onboarding_cutoff is None or event_line.timestamp <= onboarding_cutoff: + protected_events.append(event_line) + else: + evictable_events.append(event_line) + overflow = len(main_events) - self._max_events + if overflow <= 0: + return + if len(evictable_events) >= overflow: + kept_evictable = evictable_events[overflow:] + else: + kept_evictable = [] + protected_overflow = overflow - len(evictable_events) + protected_events = protected_events[protected_overflow:] + kept_events = protected_events + kept_evictable + self._write_jsonl_file(self._events_path, kept_events) + + def _get_write_state(self, path: str) -> _FileWriteState: + if path not in self._file_write_states: + self._file_write_states[path] = _FileWriteState() + return self._file_write_states[path] + + def _current_ctx(self) -> dict[str, str]: + context_field = journal_enums.JournalStorageContextField + return { + context_field.SESSION_ID.value: journal_state.get_session_id(), + context_field.APP_VERSION.value: octobot_constants.LONG_VERSION, + } + + def _should_write_ctx(self, path: str, current_ctx: dict[str, str], write_state: _FileWriteState) -> bool: + context_field = journal_enums.JournalStorageContextField + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return True + if write_state.last_ctx is None: + return True + return ( + write_state.last_ctx.get(context_field.SESSION_ID.value) != current_ctx[context_field.SESSION_ID.value] + or write_state.last_ctx.get(context_field.APP_VERSION.value) != current_ctx[context_field.APP_VERSION.value] + ) + + def _append_to_file(self, path: str, event_line: journal_models.JournalEventLine) -> None: + current_ctx = self._current_ctx() + write_state = self._get_write_state(path) + lines_to_write = [] + if self._should_write_ctx(path, current_ctx, write_state): + lines_to_write.append(self._serialize_ctx_line(current_ctx)) + lines_to_write.append(json.dumps(event_line.to_storage_dict(), separators=(",", ":"), sort_keys=True)) + with open(path, "a", encoding="utf-8") as jsonl_file: + for serialized_line in lines_to_write: + jsonl_file.write(serialized_line + "\n") + write_state.last_ctx = current_ctx + + @staticmethod + def _serialize_ctx_line(ctx: dict[str, str]) -> str: + return json.dumps( + {journal_constants.STORAGE_CTX_KEY: ctx}, + separators=(",", ":"), + sort_keys=True, + ) + + @staticmethod + def _event_dedup_key(event_line: journal_models.JournalEventLine) -> tuple: + return ( + event_line.timestamp, + event_line.event.value, + event_line.session_id, + json.dumps(event_line.attributes.to_dict(), sort_keys=True), + ) + + def _parse_jsonl_file( + self, + path: str, + *, + manifest: dict, + ) -> list[journal_models.JournalEventLine]: + if not os.path.isfile(path): + return [] + persisted_state = journal_state.load_persisted_state() + parsed_lines = [] + running_ctx: dict = {} + missing_data_warnings: set[str] = set() + with open(path, encoding="utf-8") as jsonl_file: + for line_number, raw_line in enumerate(jsonl_file, start=1): + stripped_line = raw_line.strip() + if not stripped_line: + continue + try: + parsed_line = json.loads(stripped_line) + except json.JSONDecodeError as exc: + logger.warning( + "Skipping invalid journal line %s:%s (%s: %s)", + path, + line_number, + type(exc).__name__, + exc, + ) + continue + if journal_constants.STORAGE_CTX_KEY in parsed_line: + ctx_payload = parsed_line[journal_constants.STORAGE_CTX_KEY] + if isinstance(ctx_payload, dict): + running_ctx = ctx_payload + else: + logger.warning( + "Skipping invalid journal context line %s:%s", + path, + line_number, + ) + continue + if journal_enums.JournalEventLineField.EVENT.value not in parsed_line: + logger.warning( + "Skipping unrecognized journal line %s:%s", + path, + line_number, + ) + continue + try: + parsed_lines.append( + journal_storage_hydration.hydrate_storage_event( + parsed_line, + running_ctx, + manifest, + persisted_state, + missing_data_warnings=missing_data_warnings, + ) + ) + except Exception as exc: + logger.warning( + "Skipping invalid journal line %s:%s (%s: %s)", + path, + line_number, + type(exc).__name__, + exc, + ) + return parsed_lines + + def _write_jsonl_file(self, path: str, events: list[journal_models.JournalEventLine]) -> None: + write_state = self._get_write_state(path) + if not events: + with open(path, "w", encoding="utf-8"): + pass + write_state.last_ctx = None + return + + grouped_lines: list[str] = [] + current_group_key: tuple[str, str] | None = None + current_group_events: list[journal_models.JournalEventLine] = [] + for event_line in events: + group_key = (event_line.session_id, event_line.app_version) + if group_key != current_group_key: + if current_group_events and current_group_key is not None: + grouped_lines.extend( + self._serialize_event_group(current_group_key, current_group_events) + ) + current_group_key = group_key + current_group_events = [event_line] + else: + current_group_events.append(event_line) + if current_group_events and current_group_key is not None: + grouped_lines.extend(self._serialize_event_group(current_group_key, current_group_events)) + + with open(path, "w", encoding="utf-8") as jsonl_file: + jsonl_file.write("\n".join(grouped_lines) + "\n") + + if current_group_key is not None: + last_session_id, last_app_version = current_group_key + context_field = journal_enums.JournalStorageContextField + write_state.last_ctx = { + context_field.SESSION_ID.value: last_session_id, + context_field.APP_VERSION.value: last_app_version, + } + + def _serialize_event_group( + self, + group_key: tuple[str, str], + group_events: list[journal_models.JournalEventLine], + ) -> list[str]: + session_id, app_version = group_key + context_field = journal_enums.JournalStorageContextField + serialized_lines = [ + self._serialize_ctx_line({ + context_field.SESSION_ID.value: session_id, + context_field.APP_VERSION.value: app_version, + }), + ] + for event_line in group_events: + serialized_lines.append( + json.dumps(event_line.to_storage_dict(), separators=(",", ":"), sort_keys=True) + ) + return serialized_lines + + +_default_store: JournalStore | None = None + + +def get_store(*, max_events: int | None = None) -> JournalStore: + global _default_store + if _default_store is None or (max_events is not None and _default_store._max_events != max_events): + kwargs = {} + if max_events is not None: + kwargs["max_events"] = max_events + _default_store = JournalStore(**kwargs) + return _default_store + + +def reset_default_store() -> None: + global _default_store + _default_store = None diff --git a/octobot/community/wallet_backend/community_wallet.py b/octobot/community/wallet_backend/community_wallet.py index 141e2f83d9..effa755507 100644 --- a/octobot/community/wallet_backend/community_wallet.py +++ b/octobot/community/wallet_backend/community_wallet.py @@ -32,6 +32,7 @@ InvalidPrivateKeyError, PassphraseTooShortError, WalletAlreadyExistsError, + WalletError, WalletNotFoundError, ) from octobot.community.wallet_backend.wallet_storage import ( @@ -43,6 +44,16 @@ _PBKDF2_ALG = "sha256" +def _record_wallet_operation_failure(*, operation: str, error: WalletError) -> None: + if isinstance( + error, + (InvalidPrivateKeyError, WalletAlreadyExistsError, InvalidPassphraseError), + ): + return + import octobot.community.node_journal as node_journal + node_journal.record_wallet_operation_failed(operation=operation, error=error) + + @dataclasses.dataclass class WalletInfo(commons_dataclasses.FlexibleDataclass): address: str = "" @@ -137,7 +148,9 @@ def import_wallet( try: address = sync_chain.address_from_evm_key(private_key) except Exception as err: - raise InvalidPrivateKeyError(f"Invalid EVM private key: {err}") from err + wallet_error = InvalidPrivateKeyError(f"Invalid EVM private key: {err}") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error from err return self._add_wallet_entry(private_key, address, name, passphrase, is_admin) def import_wallet_from_seed( @@ -150,7 +163,9 @@ def import_wallet_from_seed( try: wallet = sync_chain.wallet_from_mnemonic(seed.strip()) except Exception as err: - raise InvalidPrivateKeyError(f"Invalid seed phrase: {err}") from err + wallet_error = InvalidPrivateKeyError(f"Invalid seed phrase: {err}") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error from err return self._add_wallet_entry(wallet.private_key, wallet.address, name, passphrase, is_admin, seed=seed.strip()) def _add_wallet_entry( @@ -163,14 +178,20 @@ def _add_wallet_entry( seed: typing.Optional[str] = None, ) -> sync_chain.Wallet: if len(passphrase) < 8: - raise PassphraseTooShortError("Passphrase must be at least 8 characters") + wallet_error = PassphraseTooShortError("Passphrase must be at least 8 characters") + _record_wallet_operation_failure(operation="create", error=wallet_error) + raise wallet_error normalized = address.lower() with self._wallet_lock: node_wallets = self._get_node_wallets_list() if any(e.address == normalized for e in node_wallets): - raise WalletAlreadyExistsError(f"Wallet {address} already exists") + wallet_error = WalletAlreadyExistsError(f"Wallet {address} already exists") + _record_wallet_operation_failure(operation="import", error=wallet_error) + raise wallet_error if is_admin and any(e.is_admin for e in node_wallets): - raise AdminWalletAlreadyExistsError("An admin wallet already exists") + wallet_error = AdminWalletAlreadyExistsError("An admin wallet already exists") + _record_wallet_operation_failure(operation="create", error=wallet_error) + raise wallet_error entry = WalletEntry( address=normalized, name=name or None, @@ -191,9 +212,13 @@ def authenticate(self, address: str, passphrase: str) -> WalletInfo: """ entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return WalletInfo(is_admin=entry.is_admin, name=entry.name, address=entry.address) def verify_wallet_passphrase(self, address: str, passphrase: str) -> bool: @@ -206,26 +231,42 @@ def verify_wallet_passphrase(self, address: str, passphrase: str) -> bool: def decrypt_wallet_by_address(self, address: str, passphrase: str) -> sync_chain.Wallet: entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return self._wallet_from_entry(entry) def decrypt_wallet_entry_by_address(self, address: str, passphrase: str) -> WalletEntry: entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error if not _verify_passphrase_hash(passphrase, entry.passphrase_hash): - raise InvalidPassphraseError("Invalid passphrase") + wallet_error = InvalidPassphraseError("Invalid passphrase") + _record_wallet_operation_failure(operation="decrypt", error=wallet_error) + raise wallet_error return entry def get_wallet_for_bot(self, address: str) -> sync_chain.Wallet: """Return wallet without passphrase verification — for bot auto-unlock at startup.""" entry = self._find_wallet_entry(address) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="lookup", error=wallet_error) + raise wallet_error return self._wallet_from_entry(entry) + def has_wallet_for_user_id(self, user_id: str) -> bool: + for entry in self._get_node_wallets_list(): + if sync_auth.derive_user_id(entry.private_key) == user_id: + return True + return False + def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: """Return the wallet whose derived Starfish ``user_id`` matches *user_id*. @@ -239,19 +280,27 @@ def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: for entry in self._get_node_wallets_list(): if sync_auth.derive_user_id(entry.private_key) == user_id: return self._wallet_from_entry(entry) - raise WalletNotFoundError(f"Wallet not found for user_id: {user_id}") + wallet_error = WalletNotFoundError(f"Wallet not found for user_id: {user_id}") + _record_wallet_operation_failure(operation="lookup", error=wallet_error) + raise wallet_error def remove_wallet(self, address: str) -> None: normalized = address.lower() with self._wallet_lock: node_wallets = self._get_node_wallets_list() if len(node_wallets) <= 1: - raise CannotRemoveLastWalletError("Cannot remove the last wallet") + wallet_error = CannotRemoveLastWalletError("Cannot remove the last wallet") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error entry = next((e for e in node_wallets if e.address == normalized), None) if entry is None: - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error if entry.is_admin: - raise CannotRemoveAdminWalletError("Cannot remove the admin wallet") + wallet_error = CannotRemoveAdminWalletError("Cannot remove the admin wallet") + _record_wallet_operation_failure(operation="delete", error=wallet_error) + raise wallet_error self._save_node_wallets_list([e for e in node_wallets if e.address != normalized]) def rename_wallet(self, address: str, name: typing.Optional[str]) -> None: @@ -263,7 +312,9 @@ def rename_wallet(self, address: str, name: typing.Optional[str]) -> None: entry.name = name or None self._save_node_wallets_list(node_wallets) return - raise WalletNotFoundError(f"Wallet {address} not found") + wallet_error = WalletNotFoundError(f"Wallet {address} not found") + _record_wallet_operation_failure(operation="rename", error=wallet_error) + raise wallet_error def is_admin_wallet(self, address: str) -> bool: entry = self._find_wallet_entry(address) diff --git a/octobot/config/config_schema.json b/octobot/config/config_schema.json index a4aefdff21..919e2dc7b9 100644 --- a/octobot/config/config_schema.json +++ b/octobot/config/config_schema.json @@ -74,17 +74,6 @@ } } }, - "metrics": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - }, - "activity_bot_id": { - "type": "string" - } - } - }, "community-token": { "type": "string" }, diff --git a/octobot/constants.py b/octobot/constants.py index d85952686a..d5d15e6a71 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -116,8 +116,7 @@ SYNC_NAMESPACE=os.getenv("SYNC_NAMESPACE", "dk") SYNC_CHAIN_ID = os.getenv("SYNC_CHAIN_ID", "evm:8453") -ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") # has priority over ACTIVITY_TRACKER_DSN if set -ACTIVITY_TRACKER_DSN = os.getenv("ACTIVITY_TRACKER_DSN", "https://c2d3481420c3d1d5f3017c5508e7802f@o4508733888331776.ingest.de.sentry.io/4508733892132944") +ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") # Sentry error reporting only; unset disables Sentry CONFIG_COMMUNITY = "community" CONFIG_COMMUNITY_BOT_ID = "bot_id" @@ -246,6 +245,9 @@ os.getenv("CLOUD_FIRST_METRICS_UPDATE_TIME", 5) ) +# Metrics — onboarding funnel +METRICS_GENERIC_EXCHANGE_NAME = "generic" + # config types keys CONFIG_KEY = "config" TENTACLES_SETUP_CONFIG_KEY = "tentacles_setup" diff --git a/octobot/octobot.py b/octobot/octobot.py index ce2e44ed67..93340f4fc9 100644 --- a/octobot/octobot.py +++ b/octobot/octobot.py @@ -36,6 +36,8 @@ import octobot.logger as logger import octobot.community as community +import octobot.community.community_bot_stats as community_bot_stats +import octobot.community.node_journal.lifecycle as journal_startup import octobot.constants as constants import octobot.enums as enums import octobot.configuration_manager as configuration_manager @@ -78,9 +80,6 @@ def __init__(self, config: configuration.Configuration, community_authenticator= # unique aiohttp session: to be initialized from getter in a task self._aiohttp_session = None - # community if enabled - self.activity_metrics = None - # use edited config in community authentication community_config = self.get_edited_config(constants.CONFIG_KEY, dict_only=False) self.community_auth = community_authenticator or community.CommunityAuthentication.create(community_config) @@ -95,6 +94,8 @@ def __init__(self, config: configuration.Configuration, community_authenticator= # octobot_api to request the current instance self.octobot_api = octobot_api.OctoBotAPI(self) + self.community_bot_stats = community_bot_stats.CommunityBotStats(self.octobot_api) + # Logger self.logger = logging.get_logger(self.__class__.__name__) @@ -166,6 +167,10 @@ async def _post_initialize(self): self.automation = automation.Automation(self.bot_id, self.tentacles_setup_config) self._init_metadata_run_task = asyncio.create_task(self._store_run_metadata_when_available()) await self._init_profile_synchronizer() + if configuration_manager.get_distribution(self.config) is enums.OctoBotDistribution.NODE: + journal_startup.record_node_startup_succeeded( + self.get_edited_config(constants.CONFIG_KEY, dict_only=False) + ) async def _wait_for_run_data_init(self, exchange_managers, timeout): for exchange_manager in exchange_managers: @@ -243,9 +248,7 @@ async def _start_tools_tasks(self): await self.task_manager.start_tools_tasks() def _init_community(self): - self.activity_metrics = community.ActivityMetrics(self.octobot_api) - distribution = configuration_manager.get_distribution(self.config) - self.activity_metrics.setup_activity_tracking(distribution) + self.community_bot_stats.task_enabled = True async def _ensure_clock(self): if trading_api.is_trader_enabled_in_config(self.config) and constants.ENABLE_CLOCK_SYNCH: diff --git a/octobot/octobot_api.py b/octobot/octobot_api.py index 493842b9c6..7503ba4d9a 100644 --- a/octobot/octobot_api.py +++ b/octobot/octobot_api.py @@ -91,9 +91,6 @@ def get_aiohttp_session(self) -> object: def get_automation(self) -> automation.Automation: return self._octobot.automation - def get_activity_metrics(self): - return self._octobot.activity_metrics - def get_interface(self, interface_class): for interface in self._octobot.interface_producer.interfaces: if isinstance(interface, interface_class): diff --git a/octobot/task_manager.py b/octobot/task_manager.py index f921fb66ea..6d31110651 100644 --- a/octobot/task_manager.py +++ b/octobot/task_manager.py @@ -61,8 +61,8 @@ def init_async_loop(self): async def start_tools_tasks(self): task_list = [] - if self.octobot.activity_metrics: - task_list.append(self.octobot.activity_metrics.start_community_task()) + if self.octobot.community_bot_stats.task_enabled: + task_list.append(self.octobot.community_bot_stats.start_community_task()) self.octobot.async_loop = self.async_loop self.ready = True @@ -116,9 +116,8 @@ async def stop_timeout(timeout): self._process_bot_state_dump_task.cancel() self._process_bot_state_dump_task = None - # close community session - if self.octobot.activity_metrics: - stop_coroutines.append(self.octobot.activity_metrics.stop_task()) + if self.octobot.community_bot_stats.task_enabled: + stop_coroutines.append(self.octobot.community_bot_stats.stop_task()) async def _await_timeouted_gather(tasks): # await this gather to be sure to complete each stop call or timeout diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index bf42dce383..9ff94471a6 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -16,7 +16,6 @@ # License along with this library. import os import functools -import copy import typing import octobot_commons.logging as logging @@ -28,6 +27,7 @@ import octobot_commons.configuration.config_file_manager as config_file_manager import octobot_commons.configuration.config_operations as config_operations import octobot_commons.user_root_folder_provider as user_root_folder_provider +import octobot_commons.copy_util as copy_util class Configuration: @@ -90,7 +90,7 @@ def read( should_raise=should_raise, fill_missing_fields=fill_missing_fields, ) - self.config = copy.deepcopy(self._read_config) + self.config = copy_util.deepcopy(self._read_config, "read config") if activate_profile: self.load_profiles_if_possible_and_necessary() @@ -144,8 +144,10 @@ def remove_profile(self, profile_id: str) -> None: def _generate_config_from_user_config_and_profile(self): for profile_managed_element in self.profile.FULLY_MANAGED_ELEMENTS: - self.config[profile_managed_element] = copy.deepcopy( - self.profile.config[profile_managed_element] + self.config[profile_managed_element] = copy_util.deepcopy( + self.profile.config[profile_managed_element], + f"profile element {profile_managed_element!r}", + str(profile_managed_element), ) for partially_managed_element in self.profile.PARTIALLY_MANAGED_ELEMENTS: self.profile.merge_partially_managed_element_into_config( @@ -331,17 +333,6 @@ def get_tentacles_setup_config_for_package_operations(self): ) return self.get_active_tentacles_setup_config() - def get_metrics_enabled(self) -> bool: - """ - Check if metrics are enabled - :return: True if metrics are enabled - """ - return bool( - self.config.get(commons_constants.CONFIG_METRICS, {}).get( - commons_constants.CONFIG_ENABLED_OPTION, True - ) - ) - def accepted_terms(self) -> bool: """ Check if terms has been accepted @@ -505,7 +496,7 @@ def refresh_sync_profiles(self) -> None: self.profile = refreshed_profile def _get_config_without_profile_elements(self) -> dict: - filtered_config = copy.deepcopy(self.config) + filtered_config = copy_util.deepcopy(self.config, "save config filter") # do not include profile fully managed elements into filtered config for profile_managed_element in profiles.Profile.FULLY_MANAGED_ELEMENTS: filtered_config.pop(profile_managed_element, None) diff --git a/packages/commons/octobot_commons/constants.py b/packages/commons/octobot_commons/constants.py index a5cfbe8323..eb7d393317 100644 --- a/packages/commons/octobot_commons/constants.py +++ b/packages/commons/octobot_commons/constants.py @@ -220,9 +220,6 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: DEFAULT_DISTRIBUTION = "default" CONFIG_DISTRIBUTION = "distribution" -# metrics -CONFIG_METRICS = "metrics" -CONFIG_METRICS_ACTIVITY_BOT_ID = "activity_bot_id" TIMER_BEFORE_METRICS_REGISTRATION_SECONDS = 600 TIMER_BETWEEN_METRICS_UPTIME_UPDATE = float( os.getenv("TIMER_BETWEEN_METRICS_UPTIME_UPDATE", str(3600 * 4)) diff --git a/packages/commons/octobot_commons/copy_util.py b/packages/commons/octobot_commons/copy_util.py new file mode 100644 index 0000000000..4290064e48 --- /dev/null +++ b/packages/commons/octobot_commons/copy_util.py @@ -0,0 +1,78 @@ +# pylint: disable=broad-exception-caught +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy +import typing + +import octobot_commons.errors as errors + +_ROOT_FIELD_PATH = "" +_CopyValue = typing.TypeVar("_CopyValue") + + +def _build_field_path(parent_path: str, segment: str) -> str: + if not parent_path: + return segment + return f"{parent_path}.{segment}" + + +def _format_field_path(field_path: str) -> str: + return field_path or _ROOT_FIELD_PATH + + +def _find_deepcopy_failure(value, field_path: str = "") -> tuple[str, object]: + if isinstance(value, dict): + for key, nested_value in value.items(): + nested_path = _build_field_path(field_path, str(key)) + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + elif isinstance(value, list): + for index, nested_value in enumerate(value): + nested_path = f"{field_path}[{index}]" + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + elif isinstance(value, tuple): + for index, nested_value in enumerate(value): + nested_path = f"{field_path}[{index}]" + try: + copy.deepcopy(nested_value) + except Exception: + return _find_deepcopy_failure(nested_value, nested_path) + return field_path, value + + +def deepcopy(value: _CopyValue, context: str = "", root_path: str = "") -> _CopyValue: + """ + Deepcopy a value and raise CopyError with field path context on failure. + :param value: the value to deepcopy + :param context: caller context included in the error message + :param root_path: optional prefix for nested field paths + :return: the deepcopied value + """ + try: + return copy.deepcopy(value) + except Exception as err: + failing_path, failing_value = _find_deepcopy_failure(value, root_path) + formatted_path = _format_field_path(failing_path) + context_message = f" during {context}" if context else "" + raise errors.CopyError( + f"Cannot deepcopy{context_message} at field {formatted_path!r}: " + f"{type(failing_value).__name__}={failing_value!r}" + ) from err diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index 132356d8d8..3cc86c6e18 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -27,6 +27,12 @@ class RemoteConfigError(ConfigError): """ +class CopyError(Exception): + """ + Copy related Exception + """ + + class NoProfileError(Exception): """ Profile related Exception: raised when the current profile can't be found and default profile can't be loaded diff --git a/packages/commons/tests/configuration/test_configuration.py b/packages/commons/tests/configuration/test_configuration.py index 2ec9cafcc0..688f2a1315 100644 --- a/packages/commons/tests/configuration/test_configuration.py +++ b/packages/commons/tests/configuration/test_configuration.py @@ -17,11 +17,13 @@ import shutil import json import copy +import threading import pytest import mock import octobot_commons.errors as errors import octobot_commons.json_util import octobot_commons.configuration as configuration +import octobot_commons.configuration.config_file_manager as config_file_manager_module import octobot_commons.profiles as profiles import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_module @@ -447,27 +449,6 @@ def test_master_reference_path_uses_sync_data_root_for_child_process(self, confi assert config._get_master_reference_tentacles_config_file_path() == expected_path -def test_get_metrics_enabled(config): - config.config = {} - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: {} - } - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: { - constants.CONFIG_ENABLED_OPTION: True - } - } - assert config.get_metrics_enabled() is True - config.config = { - constants.CONFIG_METRICS: { - constants.CONFIG_ENABLED_OPTION: False - } - } - assert config.get_metrics_enabled() is False - - def test_accepted_terms(config): config.config = {} assert config.accepted_terms() is False @@ -768,6 +749,40 @@ def test_get_config_without_profile_elements(config): } +class TestCopyUtilOnSaveFilter: + def test_raises_copy_error_when_config_contains_non_deepcopyable_value(self, config): + config.config = {"metrics": {"runtime": threading.Lock()}} + with pytest.raises(errors.CopyError) as raised_error: + config._get_config_without_profile_elements() + assert "metrics.runtime" in str(raised_error.value) + + +class TestCopyUtilOnRead: + def test_raises_copy_error_when_read_config_contains_non_deepcopyable_value(self, config): + loaded_config = {"metrics": {"runtime": threading.Lock()}} + with mock.patch.object( + config_file_manager_module, + "load", + mock.Mock(return_value=loaded_config), + ): + with pytest.raises(errors.CopyError) as raised_error: + config.read(activate_profile=False) + assert "metrics.runtime" in str(raised_error.value) + + +class TestCopyUtilOnProfileMerge: + def test_raises_copy_error_when_profile_managed_element_contains_non_deepcopyable_value( + self, config + ): + config.profile = _load_test_profile(config) + managed_element = profiles.Profile.FULLY_MANAGED_ELEMENTS[0] + config.profile.config[managed_element] = {"runtime": threading.Lock()} + config.config = {} + with pytest.raises(errors.CopyError) as raised_error: + config._generate_config_from_user_config_and_profile() + assert str(managed_element) in str(raised_error.value) + + class TestConfigurationReadonlyProfileOverlay: def _write_readonly_master_profile( self, diff --git a/packages/commons/tests/test_copy_util.py b/packages/commons/tests/test_copy_util.py new file mode 100644 index 0000000000..042b7bfbb6 --- /dev/null +++ b/packages/commons/tests/test_copy_util.py @@ -0,0 +1,67 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import threading + +import pytest + +import octobot_commons.copy_util as copy_util +import octobot_commons.errors as errors + + +class TestDeepcopy: + def test_deepcopies_plain_nested_dict(self): + source_config = {"metrics": {"count": 1}, "enabled": True} + copied_config = copy_util.deepcopy(source_config) + assert copied_config == source_config + assert copied_config is not source_config + assert copied_config["metrics"] is not source_config["metrics"] + + def test_raises_copy_error_for_nested_dict_failure(self): + non_deepcopyable_value = threading.Lock() + source_config = {"metrics": {"runtime": non_deepcopyable_value}} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(source_config, "test context") + error_message = str(raised_error.value) + assert "metrics.runtime" in error_message + assert "lock=" in error_message + assert "test context" in error_message + + def test_raises_copy_error_for_list_index_path(self): + non_deepcopyable_value = threading.Lock() + source_config = {"items": [1, non_deepcopyable_value, 3]} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(source_config) + assert "items[1]" in str(raised_error.value) + assert "lock=" in str(raised_error.value) + + def test_raises_copy_error_with_root_path_prefix(self): + non_deepcopyable_value = threading.Lock() + source_config = {"binance": {"enabled": non_deepcopyable_value}} + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy( + source_config, + "profile element", + "crypto-currencies", + ) + error_message = str(raised_error.value) + assert "crypto-currencies.binance.enabled" in error_message + assert "lock=" in error_message + + def test_chains_original_exception_as_cause(self): + non_deepcopyable_value = threading.Lock() + with pytest.raises(errors.CopyError) as raised_error: + copy_util.deepcopy(non_deepcopyable_value) + assert isinstance(raised_error.value.__cause__, TypeError) diff --git a/packages/flow/tests/conftest.py b/packages/flow/tests/conftest.py index eb3d4dd107..bb75b42bfa 100644 --- a/packages/flow/tests/conftest.py +++ b/packages/flow/tests/conftest.py @@ -2,11 +2,26 @@ import dotenv import os +import sys + dotenv.load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), ".env")) import mock import pytest +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment + + +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield + @pytest.fixture(autouse=True) def _disable_auto_open_in_web_browser(): diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index 9a37624917..383298fa6d 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -43,6 +43,11 @@ class SchedulerQueues(enum.Enum): PORTFOLIO_HISTORY_QUEUE = "portfolio_history_queue" +class UserActionSource(enum.Enum): + SYNC = "sync" + DEBUG_API = "debug_api" + + class SignalPriorityActionPayloadKeys(enum.Enum): ACTIONS = "actions" SIGNAL = "signal" diff --git a/packages/node/octobot_node/protocol/user_actions.py b/packages/node/octobot_node/protocol/user_actions.py index 355a3147d0..68585d94a2 100644 --- a/packages/node/octobot_node/protocol/user_actions.py +++ b/packages/node/octobot_node/protocol/user_actions.py @@ -14,13 +14,23 @@ # You should have received a copy of the GNU General Public License along # with OctoBot. If not, see . import octobot_protocol.models as protocol_models + +import octobot.community.node_journal as node_journal + +import octobot_node.enums as octobot_node_enums import octobot_node.scheduler.tasks as scheduler_tasks async def execute_user_action( user_action: protocol_models.UserAction, user_id: str, + *, + source: octobot_node_enums.UserActionSource = octobot_node_enums.UserActionSource.SYNC, ) -> None: + node_journal.record_external_action_received( + user_action, + source=source.value, + ) await scheduler_tasks.trigger_user_action_workflow( user_action, user_id ) diff --git a/packages/node/octobot_node/scheduler/__init__.py b/packages/node/octobot_node/scheduler/__init__.py index 58d2eb595a..d377fc9911 100644 --- a/packages/node/octobot_node/scheduler/__init__.py +++ b/packages/node/octobot_node/scheduler/__init__.py @@ -16,16 +16,32 @@ import logging +import octobot_node.config import octobot_node.constants import octobot_node.scheduler.scheduler as scheduler_lib -import octobot_node.scheduler.workflows +import octobot_node.scheduler.workflows as scheduler_workflows import octobot_node.scheduler.workflows_version_migration as workflows_version_migration +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.recording_context as journal_recording_context + scheduler_logger = logging.getLogger(__name__) SCHEDULER: scheduler_lib.Scheduler = scheduler_lib.Scheduler() _shutdown_done = False +_scheduler_init_failure_recorded = False + + +def scheduler_init_failure_was_recorded() -> bool: + return _scheduler_init_failure_recorded + + +def _record_scheduler_init_failed(**kwargs) -> None: + global _scheduler_init_failure_recorded + _scheduler_init_failure_recorded = True + node_journal.record_scheduler_init_failed(**kwargs) def is_enabled() -> bool: @@ -36,20 +52,53 @@ def is_initialized() -> bool: return SCHEDULER.is_initialized() +def _scheduler_backend() -> journal_enums.JournalSchedulerBackend: + if octobot_node.config.settings.SCHEDULER_POSTGRES_URL: + return journal_enums.JournalSchedulerBackend.POSTGRES + return journal_enums.JournalSchedulerBackend.SQLITE + + async def initialize_scheduler(): - global _shutdown_done + global _shutdown_done, _scheduler_init_failure_recorded _shutdown_done = False + _scheduler_init_failure_recorded = False scheduler_logger.info("Initializing scheduler") - SCHEDULER.create() - octobot_node.scheduler.workflows.register_workflows() + backend = _scheduler_backend() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.DBOS_CREATE, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + SCHEDULER.create() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.REGISTER_WORKFLOWS, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + scheduler_workflows.register_workflows() if octobot_node.constants.ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION: - workflows_version_migration.migrate_stranded_workflow_versions( - target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, - ) + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.VERSION_MIGRATION, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) import octobot_node.scheduler.schedules as schedules - SCHEDULER.start() + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.DBOS_LAUNCH, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + SCHEDULER.start() # apply_schedules requires DBOS launch (sys_db); must run after start(). - await schedules.register_schedules(SCHEDULER) + with journal_recording_context.scheduler_init_phase( + init_phase=journal_enums.JournalInitPhase.REGISTER_SCHEDULES, + backend=backend, + on_failure=_record_scheduler_init_failed, + ): + await schedules.register_schedules(SCHEDULER) async def shutdown_scheduler_and_trading_signal_channel() -> None: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py index 3245162126..85bb648ed9 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py @@ -24,6 +24,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_create_account_payload( user_action: protocol_models.UserAction, @@ -66,3 +68,8 @@ async def _do_execute( ), ) self._mark_user_action_completed(user_action) + node_journal.record_account_validated_from_account( + checked_account, + self._user_id, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py index ece68d5934..363f3c555f 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/delete_account.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor +import octobot.community.node_journal as node_journal + def _get_delete_account_payload( user_action: protocol_models.UserAction, @@ -43,8 +45,13 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: delete_payload = _get_delete_account_payload(user_action) - collection_providers.AccountProvider.instance().delete_item( + account_provider = collection_providers.AccountProvider.instance() + account_provider.delete_item( self._user_id, delete_payload.id, ) self._mark_user_action_completed(user_action) + node_journal.record_account_deleted( + account_id=delete_payload.id or "", + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py index 693a36e5ea..862e61d841 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/edit_account.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_edit_account_payload( user_action: protocol_models.UserAction, @@ -61,3 +63,7 @@ async def _do_execute( checked_account, ) self._mark_user_action_completed(user_action) + node_journal.record_account_edit_succeeded( + account_id=checked_account.id or "", + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py index df4b206429..03c6f7cb4c 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/refresh_accounts.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot.community.node_journal as node_journal + def _get_refresh_accounts_payload( user_action: protocol_models.UserAction, @@ -45,11 +47,18 @@ async def _do_execute( ) -> None: refresh_payload = _get_refresh_accounts_payload(user_action) account_provider = collection_providers.AccountProvider.instance() - account_ids_to_refresh = refresh_payload.account_ids or [ - account.id for account in account_provider.list_items(self._user_id) - ] + if refresh_payload.account_ids: + account_ids_to_refresh = refresh_payload.account_ids + cached_all_accounts = None + else: + cached_all_accounts = account_provider.list_items(self._user_id) + account_ids_to_refresh = [account.id for account in cached_all_accounts] for account_id in account_ids_to_refresh: account = account_provider.get_item(self._user_id, account_id) checked_account = await account_state_updater.update_account_state(account, self._user_id) account_provider.update_item(self._user_id, checked_account) self._mark_user_action_completed(user_action) + node_journal.record_accounts_refreshed( + account_ids=list(account_ids_to_refresh), + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py index 2e442234ba..f7dc2c163c 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/create_account_auth.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor as account_auth_user_action_executor +import octobot.community.node_journal as node_journal + def _get_create_account_auth_payload( user_action: protocol_models.UserAction, @@ -51,3 +53,7 @@ async def _do_execute( account_auth_user_action_executor.with_updated_at(create_payload.configuration), ) self._mark_user_action_completed(user_action) + node_journal.record_account_auth_create_succeeded( + exchange_name=None, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py index 4f51d704b0..4955735c1e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account_auth/delete_account_auth.py @@ -20,6 +20,8 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor as account_auth_user_action_executor +import octobot.community.node_journal as node_journal + def _get_delete_account_auth_payload( user_action: protocol_models.UserAction, @@ -51,3 +53,7 @@ async def _do_execute( delete_payload.id, ) self._mark_user_action_completed(user_action) + node_journal.record_account_auth_deleted( + exchange_name=None, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index e8b645487a..b6d34e9539 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -26,6 +26,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.automation.automation_user_action_executor as automation_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory +import octobot.community.node_journal as node_journal + import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers @@ -117,14 +119,24 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: actions = self._create_automation_actions(user_action) - task = await self._create_automation_task(user_action, actions) + automation_id, task = await self._create_automation_task(user_action, actions) self.post_actions.to_create_automation_task = task self._mark_user_action_completed( user_action, created_automation_id=task.id ) + automation_configuration = self._get_automation_configuration(user_action) + stored_strategy = _load_strategy_for_automation( + self._user_id, + automation_configuration.strategy, + ) + node_journal.record_new_automation_created_from_strategy( + automation_id, + stored_strategy, + user_action_id=user_action.id, + ) - async def _create_automation_task(self, user_action, actions: list[flow_entities.AbstractActionDetails]) -> models.Task: + async def _create_automation_task(self, user_action, actions: list[flow_entities.AbstractActionDetails]) -> tuple[str, models.Task]: automation_configuration = self._get_automation_configuration(user_action) if automation_configuration.id: _validate_automation_configuration_id(automation_configuration.id) @@ -137,7 +149,7 @@ async def _create_automation_task(self, user_action, actions: list[flow_entities } if automation_configuration.id: task_fields["id"] = automation_configuration.id - return models.Task(**task_fields) + return automation_id, models.Task(**task_fields) def _get_automation_configuration(self, user_action: protocol_models.UserAction) -> protocol_models.AutomationConfiguration: create_payload = _get_create_automation_payload(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py index 2e3aefb58a..a2f191615b 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py @@ -29,6 +29,8 @@ import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util +import octobot.community.node_journal as node_journal + def _get_restart_automation_payload( user_action: protocol_models.UserAction, @@ -166,3 +168,7 @@ async def _do_execute( user_action, created_automation_id=parent_automation_id, ) + node_journal.record_automation_restarted( + automation_id=parent_automation_id, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py index 26c7625263..ac3de708ee 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/stop_automation.py @@ -21,6 +21,8 @@ import octobot_node.scheduler as scheduler_module import octobot_node.scheduler.tasks as scheduler_tasks +import octobot.community.node_journal as node_journal + def _get_stop_automation_payload( user_action: protocol_models.UserAction, @@ -71,3 +73,8 @@ async def _do_execute( actions, ) self._mark_user_action_completed(user_action) + node_journal.record_automation_stopped( + automation_id=stop_payload.id, + cancel_orders=bool(stop_payload.cancel_orders), + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py index df92edadb5..3cc286460e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/base_user_action_executor.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_action_post_actions as user_action_post_actions +import octobot.community.node_journal as node_journal + class UserActionExecutor(abc.ABC): """ @@ -92,6 +94,10 @@ async def execute( f"User action execution failed: {user_action.id}: {exc} ({exc.__class__.__name__})" ) self._apply_execution_failure(user_action, exc) + node_journal.record_executor_failure( + user_action, + error=exc, + ) raise finally: await self.after_execute(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py index 792fda9cc6..6750a0e937 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation +import octobot.community.node_journal as node_journal + def _get_create_strategy_payload( user_action: protocol_models.UserAction, @@ -55,3 +57,8 @@ async def _do_execute( create_payload.configuration, ) self._mark_user_action_completed(user_action) + node_journal.record_strategy_create_succeeded( + strategy_id=create_payload.configuration.id, + strategy=create_payload.configuration, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py index 1e0e0765b2..04fee35649 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py @@ -21,6 +21,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation +import octobot.community.node_journal as node_journal + def _get_edit_strategy_payload( user_action: protocol_models.UserAction, @@ -63,3 +65,8 @@ async def _do_execute( edit_payload.configuration, ) self._mark_user_action_completed(user_action) + node_journal.record_strategy_edit_succeeded( + strategy_id=edit_payload.configuration.id, + strategy=edit_payload.configuration, + user_action_id=user_action.id, + ) diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index 0e38c25160..04b65f423d 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -43,6 +43,8 @@ import octobot_node.scheduler.automations.signal_execution_result_util as signal_execution_result_util import octobot_node.errors as errors +import octobot.community.node_journal as node_journal + from octobot_node.scheduler import SCHEDULER # avoid circular import WORKFLOW_NAME = "execute_automation" @@ -431,6 +433,17 @@ def _finalize_postponed_iteration( f"Iteration postponed ({iteration_state.execution_error}: {iteration_state.execution_error_message}), " f"retry scheduled in {retry_delay_seconds:.0f} seconds" ) + if iteration_state.execution_error is not None: + AutomationWorkflow._record_automation_run_errored( + parsed_inputs, + error_status=iteration_state.execution_error, + error_origin="postponed_iteration", + error=Exception( + iteration_state.execution_error_message + or iteration_state.execution_error + ), + retriable=True, + ) @staticmethod async def _send_signal_execution_result_safe( @@ -683,6 +696,13 @@ def _should_continue_workflow( f"Automation stopped: unrecoverable iteration error: {progress_status.error}. " f"Iteration's last step: {progress_status.latest_step}" ) + AutomationWorkflow._record_automation_run_errored( + parsed_inputs, + error_status=progress_status.error, + error_origin="terminal_iteration", + error=Exception(progress_status.error_message or progress_status.error), + retriable=False, + ) return stop_on_error elif progress_status.should_stop: AutomationWorkflow.get_logger(parsed_inputs).info( @@ -695,6 +715,34 @@ def _should_continue_workflow( def _get_actions_summary(actions: list["octobot_flow.entities.AbstractActionDetails"], minimal: bool = False) -> str: return ", ".join([action.get_summary(minimal=minimal) for action in actions]) if actions else "" + @staticmethod + def _resolve_automation_id(parsed_inputs: params.AutomationWorkflowInputs) -> str | None: + try: + automation_state = automation_states_loader.get_automation_dict(parsed_inputs.task.content) + return automation_state.get("automation", {}).get("metadata", {}).get("automation_id") + except ValueError: + return None + + @staticmethod + def _record_automation_run_errored( + parsed_inputs: params.AutomationWorkflowInputs, + *, + error_status: str, + error_origin: str, + error: BaseException, + retriable: bool, + ) -> None: + automation_id = AutomationWorkflow._resolve_automation_id(parsed_inputs) + if automation_id is None: + return + node_journal.record_automation_run_errored( + automation_id=automation_id, + error_status=error_status, + error_origin=error_origin, + error=error, + retriable=retriable, + ) + @staticmethod def _get_failed_error_status(error: Exception) -> str: if isinstance(error, errors.WorkflowActionExecutionError): diff --git a/packages/node/tests/conftest.py b/packages/node/tests/conftest.py index b9c2767756..05b43aad56 100644 --- a/packages/node/tests/conftest.py +++ b/packages/node/tests/conftest.py @@ -1,8 +1,15 @@ import contextlib -import os - import mock +import os import pytest +import sys + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = 2 _TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS = 30.0 @@ -29,6 +36,12 @@ def mocked_local_user_configuration(): yield +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield + + @pytest.fixture(autouse=True) def _mock_local_user_configuration(): with mocked_local_user_configuration(): diff --git a/packages/node/tests/protocol/test_user_actions.py b/packages/node/tests/protocol/test_user_actions.py index 9eea42f274..21a96df024 100644 --- a/packages/node/tests/protocol/test_user_actions.py +++ b/packages/node/tests/protocol/test_user_actions.py @@ -10,6 +10,7 @@ import octobot_protocol.models as protocol_models +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.user_actions as user_actions_module _TEST_WALLET_ADDRESS = "0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb" @@ -34,10 +35,55 @@ async def test_calls_trigger_user_action_workflow_with_wallet_and_payload(self): user_actions_module.scheduler_tasks, "trigger_user_action_workflow", new_callable=mock.AsyncMock, - ) as trigger_workflow_mock: + ) as trigger_workflow_mock, mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ): await user_actions_module.execute_user_action(user_action_payload, _TEST_WALLET_ADDRESS) trigger_workflow_mock.assert_awaited_once_with(user_action_payload, _TEST_WALLET_ADDRESS) + @pytest.mark.asyncio + async def test_sync_source_emits_entry_onboarding_milestones(self): + user_action_payload = _minimal_user_action(action_identifier="ua-entry-check") + with mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ) as entry_mock, mock.patch.object( + user_actions_module.scheduler_tasks, + "trigger_user_action_workflow", + new_callable=mock.AsyncMock, + ): + await user_actions_module.execute_user_action( + user_action_payload, + _TEST_WALLET_ADDRESS, + source=octobot_node_enums.UserActionSource.SYNC, + ) + entry_mock.assert_called_once_with( + user_action_payload, + source=octobot_node_enums.UserActionSource.SYNC.value, + ) + + @pytest.mark.asyncio + async def test_debug_api_emits_entry_onboarding_milestones_with_source(self): + user_action_payload = _minimal_user_action(action_identifier="ua-debug-check") + with mock.patch.object( + user_actions_module.node_journal, + "record_external_action_received", + ) as entry_mock, mock.patch.object( + user_actions_module.scheduler_tasks, + "trigger_user_action_workflow", + new_callable=mock.AsyncMock, + ): + await user_actions_module.execute_user_action( + user_action_payload, + _TEST_WALLET_ADDRESS, + source=octobot_node_enums.UserActionSource.DEBUG_API, + ) + entry_mock.assert_called_once_with( + user_action_payload, + source=octobot_node_enums.UserActionSource.DEBUG_API.value, + ) + @pytest.mark.asyncio async def test_surfaces_scheduler_not_initialized_as_runtime_error(self): user_action_payload = _minimal_user_action(action_identifier="ua-not-init") diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py index 5a1926e327..3989067e92 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py @@ -229,3 +229,105 @@ async def test_fails_when_authentication_details_missing_for_live_account(self): expect_error_details=True, expected_error_message=protocol_models.AccountActionResultErrorMessage.ACCOUNT_AUTHENTICATION_DETAILS_NOT_FOUND, ) + + +class TestCreateAccountActionExecutorRecordAccountValidated: + @pytest.mark.asyncio + async def test_emits_account_validated_via_node_journal(self): + account_model = account_executor_test_utils.minimal_exchange_account( + account_id="validated-acc", + is_simulated=False, + ) + validated_account = account_model.model_copy( + update={ + "state": protocol_models.AccountState( + status=protocol_models.AccountStatus.VALID, + ), + }, + ) + inner = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=account_model, + ) + user_action = protocol_models.UserAction( + id="ua-validated", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() + metrics_config = mock.Mock() + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), + mock.patch.object( + account_state_updater_module, + "update_account_state", + new=mock.AsyncMock(return_value=validated_account), + ), + mock.patch( + "octobot.community.node_journal.record_account_validated_from_account", + ) as record_account_validated_mock, + ): + executor = create_account_executor.CreateAccountActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + record_account_validated_mock.assert_called_once_with( + validated_account, + account_executor_test_utils.WALLET_ADDRESS, + user_action_id="ua-validated", + ) + + @pytest.mark.asyncio + async def test_invalid_real_emits_no_account_validated(self): + account_model = account_executor_test_utils.minimal_exchange_account( + account_id="invalid-acc", + is_simulated=False, + ) + invalid_account = account_model.model_copy( + update={ + "state": protocol_models.AccountState( + status=protocol_models.AccountStatus.INVALID, + ), + }, + ) + inner = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=account_model, + ) + user_action = protocol_models.UserAction( + id="ua-invalid", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() + metrics_config = mock.Mock() + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), + mock.patch.object( + account_state_updater_module, + "update_account_state", + new=mock.AsyncMock(return_value=invalid_account), + ), + mock.patch( + "octobot.community.node_journal.record_account_validated_from_account", + ) as record_account_validated_mock, + ): + executor = create_account_executor.CreateAccountActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + record_account_validated_mock.assert_called_once() diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py index 1ec943317f..f3d3aafd9b 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_delete_account.py @@ -36,12 +36,23 @@ async def test_calls_provider_delete_with_wallet_and_id(self): ) user_action = protocol_models.UserAction(id="ua-del", configuration=account_executor_test_utils.wrap_configuration(inner)) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.AccountProvider.instance", - return_value=provider_mock, + deleted_account = account_executor_test_utils.minimal_exchange_account(account_id="del-1") + provider_mock.get_item.return_value = deleted_account + provider_mock.list_items.return_value = [] + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.delete_account.node_journal.record_account_deleted", + ) as record_account_deleted_mock, ): executor = delete_account_executor.DeleteAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_account_deleted_mock.assert_called_once() + assert record_account_deleted_mock.call_args.kwargs["account_id"] == "del-1" + assert record_account_deleted_mock.call_args.kwargs["user_action_id"] == "ua-del" provider_mock.delete_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, "del-1") provider_assertions.assert_user_action_terminal_state( user_action=user_action, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py index a85b9fac70..a7c5909b3a 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_edit_account.py @@ -48,9 +48,15 @@ async def test_calls_provider_update_with_wallet_and_account(self): "update_account_state", new=mock.AsyncMock(return_value=account_model), ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.edit_account.node_journal.record_account_edit_succeeded", + ) as record_account_edit_succeeded_mock, ): executor = edit_account_executor.EditAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_account_edit_succeeded_mock.assert_called_once() + assert record_account_edit_succeeded_mock.call_args.kwargs["account_id"] == "edit-acc" + assert record_account_edit_succeeded_mock.call_args.kwargs["user_action_id"] == "ua-edit" provider_mock.update_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, account_model) provider_assertions.assert_user_action_terminal_state( user_action=user_action, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py index 821e9b89de..78df5920a1 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_refresh_accounts.py @@ -65,9 +65,16 @@ async def test_updates_all_accounts_when_ids_are_not_provided(self): "update_account_state", new=mock.AsyncMock(side_effect=[checked_first_account, checked_second_account]), ) as check_mock, + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.refresh_accounts.node_journal.record_accounts_refreshed", + ) as record_accounts_refreshed_mock, ): executor = refresh_accounts_executor.RefreshAccountsActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) + record_accounts_refreshed_mock.assert_called_once_with( + account_ids=["acc-1", "acc-2"], + user_action_id="ua-refresh-all", + ) provider_mock.list_items.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS) assert provider_mock.get_item.call_count == 2 provider_mock.update_item.assert_has_calls( @@ -102,6 +109,7 @@ async def test_updates_only_requested_accounts(self): user_action = protocol_models.UserAction(id="ua-refresh-one", configuration=account_executor_test_utils.wrap_configuration(refresh_inner)) provider_mock = mock.Mock() provider_mock.get_item.return_value = account_model + provider_mock.list_items.return_value = [account_model] with ( mock.patch( "octobot_sync.sync.collection_providers.AccountProvider.instance", diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py index d83ead7805..16fe080afb 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_create_account_auth.py @@ -56,11 +56,18 @@ async def test_calls_provider_create_with_wallet_and_authentication(self): "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.account_auth_user_action_executor.timestamp_util.utc_now_datetime", return_value=_FIXED_TIMESTAMP, ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.create_account_auth.node_journal.record_account_auth_create_succeeded", + ) as record_account_auth_create_succeeded_mock, ): executor = create_account_auth_executor.CreateAccountAuthActionExecutor( account_auth_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_account_auth_create_succeeded_mock.assert_called_once_with( + exchange_name=None, + user_action_id="ua-create-auth", + ) provider_mock.create_item.assert_called_once_with( account_auth_executor_test_utils.WALLET_ADDRESS, authentication_model.model_copy(update={"updated_at": _FIXED_TIMESTAMP}), diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py index 6d320d9ff8..718fb9ac42 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account_auth/test_delete_account_auth.py @@ -39,14 +39,23 @@ async def test_calls_provider_delete_with_wallet_and_id(self): configuration=account_auth_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.AccountAuthenticationProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountAuthenticationProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account_auth.delete_account_auth.node_journal.record_account_auth_deleted", + ) as record_account_auth_deleted_mock, ): executor = delete_account_auth_executor.DeleteAccountAuthActionExecutor( account_auth_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_account_auth_deleted_mock.assert_called_once_with( + exchange_name=None, + user_action_id="ua-del-auth", + ) provider_mock.delete_item.assert_called_once_with( account_auth_executor_test_utils.WALLET_ADDRESS, "del-auth-1", diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py index ebe2fc7165..21d2041a99 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py @@ -1180,7 +1180,7 @@ def test_create_automation_task_wraps_actions_in_state_envelope(self): strategy_mock.return_value.get_item.return_value = stored actions = executor._create_automation_actions(user_action) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) assert task.name == automation_name _assert_task_content_matches_actions( task=task, @@ -1308,7 +1308,8 @@ def test_configuration_id_used_for_automation_identity(self): protocol_account=account, strategy_reference=strat_ref, ) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) + assert resolved_automation_id == configuration_automation_id assert task.id == configuration_automation_id _assert_task_content_matches_actions( task=task, @@ -1381,7 +1382,7 @@ def test_missing_configuration_id_keeps_legacy_identity_split(self): protocol_account=_minimal_exchange_account(account_id="acc-1"), strategy_reference=strat_ref, ) - task = asyncio.run(executor._create_automation_task(user_action, actions)) + _resolved_automation_id, task = asyncio.run(executor._create_automation_task(user_action, actions)) assert task.id != user_action.id _assert_task_content_matches_actions( task=task, @@ -1454,3 +1455,37 @@ async def test_execute_fails_with_invalid_configuration_id(self): expect_error_details=True, expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_AUTOMATION_ID, ) + + +class TestCreateAutomationActionExecutorRecordNewAutomationCreated: + @pytest.mark.asyncio + async def test_calls_record_new_automation_created_on_success(self): + idx = trading_tentacles_test_utils.index_trading_configuration( + coins=[("BTC", 1.0)], + rebalance_trigger_min_percent=5.0, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="metrics-automation", + strategy_reference=strat_ref, + account_id="acc-1", + ), + ) + user_action = _user_action_with_context(action_id="ua-metrics", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + stored = _stored_strategy_matching_reference(strat_ref, idx) + with mock.patch(_ACCOUNT_PROVIDER_INSTANCE_PATCH) as account_mock, mock.patch( + _STRATEGY_PROVIDER_INSTANCE_PATCH, + ) as strategy_mock, mock.patch( + "octobot.community.node_journal.record_new_automation_created_from_strategy", + ) as record_new_automation_mock: + _stub_account_provider(account_mock, _minimal_exchange_account(account_id="acc-1")) + strategy_mock.return_value.get_item.return_value = stored + await executor.execute(user_action) + record_new_automation_mock.assert_called_once_with( + user_action.id, + stored, + user_action_id="ua-metrics", + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py index a43143c00a..55db5e1f23 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py @@ -167,9 +167,16 @@ async def test_execute_enqueues_task_from_latest_output(self): new_callable=mock.AsyncMock, return_value=terminal_workflow, ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.node_journal.record_automation_restarted", + ) as record_automation_restarted_mock, ): await executor.execute(user_action) + record_automation_restarted_mock.assert_called_once_with( + automation_id=_PARENT_AUTOMATION_ID, + user_action_id="ua-restart-1", + ) scheduled_task = executor.post_actions.to_create_automation_task assert scheduled_task is not None assert scheduled_task.id == f"{_PARENT_AUTOMATION_ID}_1" diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py index 034d75a03f..211ae3f7f9 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py @@ -85,6 +85,9 @@ async def test_execute_sends_stop_actions_to_active_automation(self): "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", new_callable=mock.AsyncMock, ) as send_actions_mock, + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.node_journal.record_automation_stopped", + ) as record_automation_stopped_mock, ): await executor.execute(user_action) @@ -98,6 +101,11 @@ async def test_execute_sends_stop_actions_to_active_automation(self): } ], ) + record_automation_stopped_mock.assert_called_once_with( + automation_id="00000000-0000-4000-8000-000000000001", + cancel_orders=False, + user_action_id="ua-stop-1", + ) provider_assertions.assert_user_action_terminal_state( user_action=user_action, expected_status=protocol_models.UserActionStatus.COMPLETED, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py index 95e5fc8859..9c2d2d3e10 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_create_strategy.py @@ -40,14 +40,24 @@ async def test_calls_provider_create_with_wallet_and_strategy(self): configuration=strategy_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.StrategyProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.strategy.create_strategy.node_journal.record_strategy_create_succeeded", + ) as record_strategy_create_succeeded_mock, ): executor = create_strategy_executor.CreateStrategyActionExecutor( strategy_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_strategy_create_succeeded_mock.assert_called_once_with( + strategy_id="new-strategy", + strategy=strategy_model, + user_action_id="ua-create-strategy", + ) provider_mock.create_item.assert_called_once_with( strategy_executor_test_utils.WALLET_ADDRESS, strategy_model, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py index 653a36594f..13377e8eea 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/test_edit_strategy.py @@ -44,14 +44,24 @@ async def test_calls_provider_update_with_wallet_and_strategy(self): configuration=strategy_executor_test_utils.wrap_configuration(inner), ) provider_mock = mock.Mock() - with mock.patch( - "octobot_sync.sync.collection_providers.StrategyProvider.instance", - return_value=provider_mock, + with ( + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.strategy.edit_strategy.node_journal.record_strategy_edit_succeeded", + ) as record_strategy_edit_succeeded_mock, ): executor = edit_strategy_executor.EditStrategyActionExecutor( strategy_executor_test_utils.WALLET_ADDRESS, ) await executor.execute(user_action) + record_strategy_edit_succeeded_mock.assert_called_once_with( + strategy_id="edit-strategy", + strategy=strategy_model, + user_action_id="ua-edit-strategy", + ) provider_mock.update_item.assert_called_once_with( strategy_executor_test_utils.WALLET_ADDRESS, strategy_model, diff --git a/packages/services/octobot_services/services/service_factory.py b/packages/services/octobot_services/services/service_factory.py index 2bde86b9b6..e44e0c8b02 100644 --- a/packages/services/octobot_services/services/service_factory.py +++ b/packages/services/octobot_services/services/service_factory.py @@ -18,6 +18,8 @@ import octobot_commons.logging as logging +import octobot.configuration_manager as configuration_manager +import octobot.enums as octobot_enums import octobot_services.constants as constants import octobot_services.services as services @@ -85,6 +87,22 @@ async def _perform_checkup(self, service) -> bool: self.logger.warning(f"{service.get_name()} initial checkup failed.") except Exception as e: self.logger.exception(e, True, f"{service.get_name()} preparation produced the following error: {e}") + if ( + configuration_manager.get_distribution(self.config) is octobot_enums.OctoBotDistribution.NODE + and service.get_type() == constants.CONFIG_NODE_API + ): + import octobot_node.scheduler as scheduler_module + import octobot.community.node_journal.enums as journal_enums + import octobot.community.node_journal.recording_context as journal_recording_context + import octobot.community.node_journal.lifecycle as journal_startup + journal_recording_context.node_api_startup_failure( + error=e, + startup_phase=journal_enums.JournalStartupPhase.NODE_API_START, + force_exit=False, + config=self.config, + scheduler_init_failure_was_recorded=scheduler_module.scheduler_init_failure_was_recorded, + record_node_startup_failed=journal_startup.record_node_startup_failed, + ) return False @staticmethod diff --git a/packages/services/tests/conftest.py b/packages/services/tests/conftest.py new file mode 100644 index 0000000000..2507677251 --- /dev/null +++ b/packages/services/tests/conftest.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Services +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import sys + +import pytest + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_test_support import disabled_node_journal_environment + + +@pytest.fixture(autouse=True) +def disable_node_journal(): + with disabled_node_journal_environment(): + yield diff --git a/packages/sync/octobot_sync/app.py b/packages/sync/octobot_sync/app.py index d3495939af..7779813e9c 100644 --- a/packages/sync/octobot_sync/app.py +++ b/packages/sync/octobot_sync/app.py @@ -31,6 +31,8 @@ import octobot_sync.constants as constants import octobot_sync.sync as sync +import octobot.community.node_journal as node_journal + _VERSION_MARKER = f"/{constants.STARFISH_SERVER_MAJOR_VERSION}/" @@ -121,9 +123,23 @@ def _build_role_resolver(is_allowed_user_id: Callable[[str], bool] | None): return resolver async def gated_resolver(request): - result = await resolver(request) + try: + result = await resolver(request) + except CapAuthError as err: + node_journal.record_sync_read_failed( + collection="user-data", + failure_reason="cap_auth", + error=err, + ) + raise if result.identity and not is_allowed_user_id(result.identity): - raise CapAuthError(403, "user not allowed") + auth_error = CapAuthError(403, "user not allowed") + node_journal.record_sync_read_failed( + collection="user-data", + failure_reason="cap_auth", + error=auth_error, + ) + raise auth_error return result return gated_resolver diff --git a/packages/sync/octobot_sync/server.py b/packages/sync/octobot_sync/server.py index ac6a5ead4b..83ca71d71e 100644 --- a/packages/sync/octobot_sync/server.py +++ b/packages/sync/octobot_sync/server.py @@ -38,6 +38,10 @@ import octobot_sync.crypto as sync_crypto import octobot_sync.enums as enums import octobot_sync.errors as errors +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.recording_context as journal_recording_context # Re-exported for callers (e.g. node_api) that build the userId allowlist from # the node's own wallet keys — must use the same derivation as the client. @@ -46,6 +50,7 @@ import octobot_protocol.models as protocol_models import octobot_node.protocol.user_actions as user_actions_protocol +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.user_data as user_data_protocol import octobot_node.protocol.debug as debug_protocol import octobot_node.protocol.accounts as accounts_protocol @@ -147,7 +152,11 @@ async def _user_actions_after_write(event: WriteEvent) -> None: if action is None: return try: - await user_actions_protocol.execute_user_action(action, identity) + await user_actions_protocol.execute_user_action( + action, + identity, + source=octobot_node_enums.UserActionSource.SYNC, + ) except Exception as exc: _get_logger().exception( exc, True, f"Unexpected error executing user action: {action.id}: {exc}" @@ -175,85 +184,112 @@ def _get_opaque_store() -> FilesystemObjectStore: return _opaque_store +def _sync_read_failure_reason(error: BaseException) -> str: + if isinstance(error, errors.OctobotSyncWalletNotFoundError): + return "wallet_not_found" + if isinstance(error, errors.OctobotSyncIdentityMissingError): + return "identity_missing" + if isinstance(error, collection_errors.CollectionStorageError): + return "storage_error" + return "other" + + +def _sync_read_failure_collection(context: StoreContext | None) -> str: + if context and context.collection: + return context.collection + return "unknown" + + async def get_data(key: str, context: StoreContext | None = None) -> str | None: # called when client pulls - collection = _get_collection(context) - plaintext = None - already_encrypted_payload = None - match collection: - case enums.Collections.USER_DATA.value: - user_data_state = await user_data_protocol.get_user_data_state( - _get_identity(context) - ) - plaintext = user_data_state.to_json() - case enums.Collections.USER_ACCOUNTS.value: - encrypted_blob = accounts_protocol.get_accounts_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_AUTH.value: - encrypted_blob = accounts_auth_protocol.get_accounts_authentication_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_TRADING.value: - encrypted_blob = accounts_trading_protocol.get_account_trading_state_encrypted( - _get_identity(context), - _get_account_id(context), - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACCOUNTS_HISTORY.value: - history_state = await accounts_history_protocol.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - _get_account_id(context), - ) - plaintext = history_state.to_json() - case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_REAL.value: - history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - is_simulated=False, - ) - plaintext = history_state.to_json() - case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_SIMULATED.value: - history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( - _get_identity(context), - is_simulated=True, - ) - plaintext = history_state.to_json() - case enums.TemporaryCollections.TEMP_USER_STRATEGIES.value: - encrypted_blob = strategies_protocol.get_strategies_state_encrypted( - _get_identity(context) - ) - already_encrypted_payload = json.dumps(encrypted_blob) - case enums.Collections.USER_ACTIONS.value: - # reading user actions should always return an empty list - actions_state = protocol_models.UserActionsState( - version=sync_constants.USER_ACTIONS_STATE_VERSION, - user_actions=[] - ) - plaintext = actions_state.to_json() - case enums.Collections.DEBUG.value: - debug_state = await debug_protocol.get_debug_state( - _get_identity(context) + with journal_recording_context.sync_read_operation( + resolve_collection=lambda: _sync_read_failure_collection(context), + resolve_failure_reason=_sync_read_failure_reason, + ): + collection = _get_collection(context) + plaintext = None + already_encrypted_payload = None + match collection: + case enums.Collections.USER_DATA.value: + user_data_state = await user_data_protocol.get_user_data_state( + _get_identity(context) + ) + plaintext = user_data_state.to_json() + case enums.Collections.USER_ACCOUNTS.value: + encrypted_blob = accounts_protocol.get_accounts_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_AUTH.value: + encrypted_blob = accounts_auth_protocol.get_accounts_authentication_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_TRADING.value: + encrypted_blob = accounts_trading_protocol.get_account_trading_state_encrypted( + _get_identity(context), + _get_account_id(context), + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACCOUNTS_HISTORY.value: + history_state = await accounts_history_protocol.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + _get_account_id(context), + ) + plaintext = history_state.to_json() + case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_REAL.value: + history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + is_simulated=False, + ) + plaintext = history_state.to_json() + case enums.Collections.USER_ACCOUNTS_HISTORY_AGGREGATED_SIMULATED.value: + history_state = await accounts_history_protocol.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + _get_identity(context), + is_simulated=True, + ) + plaintext = history_state.to_json() + case enums.TemporaryCollections.TEMP_USER_STRATEGIES.value: + encrypted_blob = strategies_protocol.get_strategies_state_encrypted( + _get_identity(context) + ) + already_encrypted_payload = json.dumps(encrypted_blob) + case enums.Collections.USER_ACTIONS.value: + # reading user actions should always return an empty list + actions_state = protocol_models.UserActionsState( + version=sync_constants.USER_ACTIONS_STATE_VERSION, + user_actions=[] + ) + plaintext = actions_state.to_json() + case enums.Collections.DEBUG.value: + debug_state = await debug_protocol.get_debug_state( + _get_identity(context) + ) + plaintext = debug_state.to_json() + case _: + # Opaque storage: collections with no protocol bridge are persisted + # as client-encrypted ciphertext and the node never decrypts them. + ciphertext = await _get_opaque_store().get_string(key) + if ciphertext is None: + return None + # Stored bytes are already the client's ciphertext — hash them + # directly so it stays stable until the next push overwrites it. + return _wrap_as_stored_document(ciphertext, ciphertext) + if already_encrypted_payload is not None: + # Pre-encrypted payload (USER_ACCOUNTS): hash the encrypted JSON itself — + # it is deterministic (read from disk) so the hash stays stable. + result = _wrap_as_stored_document(already_encrypted_payload, already_encrypted_payload) + elif plaintext is None: + return None + else: + encrypted = _encrypt(plaintext, _get_identity(context), collection) + result = _wrap_as_stored_document(encrypted, plaintext) + if collection == enums.Collections.USER_DATA.value: + node_journal.on_user_data_pull_succeeded( + sync_user_id=_get_identity(context), + collection=collection, ) - plaintext = debug_state.to_json() - case _: - # Opaque storage: collections with no protocol bridge are persisted - # as client-encrypted ciphertext and the node never decrypts them. - ciphertext = await _get_opaque_store().get_string(key) - if ciphertext is None: - return None - # Stored bytes are already the client's ciphertext — hash them - # directly so it stays stable until the next push overwrites it. - return _wrap_as_stored_document(ciphertext, ciphertext) - if already_encrypted_payload is not None: - # Pre-encrypted payload (USER_ACCOUNTS): hash the encrypted JSON itself — - # it is deterministic (read from disk) so the hash stays stable. - return _wrap_as_stored_document(already_encrypted_payload, already_encrypted_payload) - if plaintext is None: - return None - encrypted = _encrypt(plaintext, _get_identity(context), collection) - return _wrap_as_stored_document(encrypted, plaintext) + return result async def put_data(key: str, body: str, context: StoreContext | None = None) -> None: # Opaque storage: persist the client ciphertext as-is. The node never diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index 1ddd09d511..f22424caca 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -32,6 +32,10 @@ import octobot_sync.sync.collection_backend.state_model as state_model import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.recording_context as journal_recording_context + _MISSING_FILE_CHECKSUM = "" @@ -125,11 +129,21 @@ def _decrypt( ] = None, ) -> state_model.StateModel: try: - plaintext_bytes = sync_crypto.decrypt_blob_dict_to_bytes( - blob, - wallet_private_key, - self.collection, - ) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_DECRYPT_FAILED, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + plaintext_bytes = sync_crypto.decrypt_blob_dict_to_bytes( + blob, + wallet_private_key, + self.collection, + ) except sync_errors.OctobotSyncCryptoFormatError as err: raise collection_errors.CollectionFileFormatError( f"{self.collection} blob: {err}" @@ -140,15 +154,20 @@ def _decrypt( ) from err try: - if strict: - decrypted_state = state_model.from_json(plaintext_bytes.decode("utf-8")) - else: - decrypted_state = tolerant_state_loading.TolerantStateLoader( - state_model, - collection=self.collection, - model_sanitizers=model_sanitizers, - model_fallbacks=model_fallbacks, - ).from_json(plaintext_bytes.decode("utf-8")) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + if strict: + decrypted_state = state_model.from_json(plaintext_bytes.decode("utf-8")) + else: + decrypted_state = tolerant_state_loading.TolerantStateLoader( + state_model, + collection=self.collection, + model_sanitizers=model_sanitizers, + model_fallbacks=model_fallbacks, + ).from_json(plaintext_bytes.decode("utf-8")) except Exception as err: raise collection_errors.CollectionFileFormatError( f"Decrypted {self.collection} payload is not valid JSON: {err}" @@ -201,9 +220,15 @@ def _read_blob(self, storage_key: str) -> dict[str, typing.Any]: with open(path, "r", encoding="utf-8") as handle: raw = json.load(handle) if not isinstance(raw, dict): - raise collection_errors.CollectionFileFormatError( + format_error = collection_errors.CollectionFileFormatError( f"{self.collection} file must contain an encrypted blob object" ) + with journal_recording_context.sync_storage_error( + event=journal_events.NodeJournalEvent.SYNC_STORAGE_FORMAT_ERROR, + collection=self.collection, + provider=journal_enums.SyncStorageProvider.LOCAL, + ): + raise format_error return raw def load_items_encrypted(self, storage_key: str) -> dict[str, str]: diff --git a/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py b/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py index 109872ceaf..5851317d29 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py +++ b/packages/sync/octobot_sync/sync/collection_backend/tolerant_state_loading.py @@ -21,6 +21,9 @@ import octobot_sync.sync.collection_backend.state_model as state_model +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.events as journal_events + logger = commons_logging.get_logger("TolerantStateLoading") @@ -50,6 +53,14 @@ def __init__( self.model_sanitizers = model_sanitizers or {} self.model_fallbacks = model_fallbacks or {} + def _record_schema_recovery(self, recovery_action: str) -> None: + node_journal.record_sync_storage_event( + journal_events.NodeJournalEvent.SYNC_STORAGE_SCHEMA_RECOVERY, + collection=self.collection, + provider="local", + recovery_action=recovery_action, + ) + def from_json(self, json_str: str) -> state_model.StateModel: if self.state_class is None: raise ValueError("state_class is required for from_json") @@ -98,9 +109,11 @@ def from_dict( parsed_state_dict, ) try: - return _parse_model_strict(self.state_class, sanitized_state_dict) + healed_state = _parse_model_strict(self.state_class, sanitized_state_dict) except Exception as retry_error: raise retry_error from strict_error + self._record_schema_recovery("sanitize_state") + return healed_state def model_from_dict_lenient( self, @@ -118,6 +131,7 @@ def model_from_dict_lenient( context, type(raw_dict).__name__, ) + self._record_schema_recovery("skip_item") return None raise ValueError( f"Expected dict for {model_class.__name__} in {context}, " @@ -133,13 +147,16 @@ def model_from_dict_lenient( return _parse_model_strict(model_class, sanitized_dict) except Exception as retry_error: try: - return self._rebuild_model_from_sanitized_dict( + rebuilt_model = self._rebuild_model_from_sanitized_dict( model_class, sanitized_dict, context=context, ) except Exception as rebuild_error: retry_error = rebuild_error + else: + self._record_schema_recovery("rebuild_item") + return rebuilt_model fallback_factory = self.model_fallbacks.get(model_class) if fallback_factory is not None: logger.warning( @@ -149,6 +166,7 @@ def model_from_dict_lenient( context, retry_error, ) + self._record_schema_recovery("fallback_item") return fallback_factory() if allow_skip: logger.warning( @@ -158,6 +176,7 @@ def model_from_dict_lenient( context, retry_error, ) + self._record_schema_recovery("skip_item") return None raise retry_error from strict_error diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index 5cb37641eb..58c6a2e8bf 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -18,7 +18,6 @@ import typing import octobot.community.authentication as community_authentication -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_commons.logging as commons_logging import octobot_commons.singleton.singleton_class as singleton_class import octobot_sync.constants as sync_constants @@ -139,13 +138,11 @@ def list_collectable_wallet_ids(self) -> list[str]: community_auth = community_authentication.CommunityAuthentication.instance() collectable_wallet_ids = [] for wallet_id in self.list_registered_wallet_ids(): - try: - community_auth.get_wallet_by_user_id(wallet_id) - except wallet_backend_errors.WalletNotFoundError: + if community_auth.has_wallet_for_user_id(wallet_id): + collectable_wallet_ids.append(wallet_id) + else: logger.debug( "Skipping wallet %s: not registered locally", wallet_id, ) - continue - collectable_wallet_ids.append(wallet_id) return collectable_wallet_ids diff --git a/packages/sync/tests/conftest.py b/packages/sync/tests/conftest.py index 6c1a4297f5..ebced54a9a 100644 --- a/packages/sync/tests/conftest.py +++ b/packages/sync/tests/conftest.py @@ -16,6 +16,52 @@ """Shared test fixtures.""" +import os +import sys + +import pytest + +_OCTOBOT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_TESTS_ROOT = os.path.join(_OCTOBOT_ROOT, "tests") +if _TESTS_ROOT not in sys.path: + sys.path.insert(0, _TESTS_ROOT) + +from test_utils.journal_integration_fixtures import ( + isolated_journal_environment_context, + journal_persisted_state, + journal_user_root, + surface_journal_errors_context, +) +from test_utils.journal_test_support import ( + disabled_node_journal_environment, + enabled_node_journal_environment, +) + + +def _is_sync_journal_integration_test(request) -> bool: + return request.node.path.name.endswith("_journal.py") + + +@pytest.fixture(autouse=True) +def disable_node_journal(request): + if _is_sync_journal_integration_test(request): + yield + return + with disabled_node_journal_environment(): + yield + + +@pytest.fixture(autouse=True) +def sync_journal_integration_autouse(request): + if not _is_sync_journal_integration_test(request): + yield + return + journal_user_root = request.getfixturevalue("journal_user_root") + with enabled_node_journal_environment(): + with isolated_journal_environment_context(str(journal_user_root)): + with surface_journal_errors_context(): + yield + class MemoryObjectStore: """Minimal AbstractObjectStore for testing.""" diff --git a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py index beaa56c092..b109c60c5d 100644 --- a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py +++ b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py @@ -17,9 +17,10 @@ import mock import octobot.community.authentication as community_authentication -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_sync.sync.collection_providers.user_account_provider as account_provider_module +_JOURNAL_PATCH = "octobot.community.node_journal.record_wallet_operation_failed" + class TestListCollectableWalletIds: def test_returns_only_wallets_registered_locally(self): @@ -29,11 +30,7 @@ def test_returns_only_wallets_registered_locally(self): "wallet-missing", ] community_auth = mock.Mock() - community_auth.get_wallet_by_user_id.side_effect = lambda wallet_id: ( - mock.Mock() - if wallet_id == "wallet-known" - else (_ for _ in ()).throw(wallet_backend_errors.WalletNotFoundError("missing")) - ) + community_auth.has_wallet_for_user_id.side_effect = lambda wallet_id: wallet_id == "wallet-known" with mock.patch.object( community_authentication.CommunityAuthentication, "instance", @@ -42,6 +39,23 @@ def test_returns_only_wallets_registered_locally(self): result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) assert result == ["wallet-known"] + def test_skips_orphan_wallet_ids_without_journaling(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [ + "wallet-known", + "wallet-missing", + ] + community_auth = mock.Mock() + community_auth.has_wallet_for_user_id.side_effect = lambda wallet_id: wallet_id == "wallet-known" + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=community_auth, + ), mock.patch(_JOURNAL_PATCH) as record_mock: + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == ["wallet-known"] + record_mock.assert_not_called() + def test_returns_empty_when_no_registered_wallets(self): provider = mock.Mock(spec=account_provider_module.AccountProvider) provider.list_registered_wallet_ids.return_value = [] diff --git a/packages/sync/tests/test_server.py b/packages/sync/tests/test_server.py index cfbd70d3e3..f3a8da044a 100644 --- a/packages/sync/tests/test_server.py +++ b/packages/sync/tests/test_server.py @@ -11,6 +11,8 @@ import octobot_sync.enums as enums import octobot_sync.errors as errors +import octobot_node.enums as octobot_node_enums + from starfish_server.storage.base import StoreContext from starfish_server.storage.s3 import S3ObjectStore from starfish_server.storage.filesystem import FilesystemObjectStore @@ -370,7 +372,11 @@ async def test_user_actions_executes_appended_action(self): mock_proto.execute_user_action = mock.AsyncMock() await server._user_actions_after_write(event) mock_pm.UserAction.from_json.assert_called_once_with(plain_body) - mock_proto.execute_user_action.assert_awaited_once_with(action, "0xwallet") + mock_proto.execute_user_action.assert_awaited_once_with( + action, + "0xwallet", + source=octobot_node_enums.UserActionSource.SYNC, + ) @pytest.mark.asyncio async def test_user_actions_logs_exception_on_failure(self): diff --git a/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py b/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py new file mode 100644 index 0000000000..5d2fcd1070 --- /dev/null +++ b/packages/sync/tests/test_sync_read_failed_cap_auth_journal.py @@ -0,0 +1,55 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from starfish_server.router.cap_resolver import CapAuthError + +import octobot_sync.app as sync_app + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module + + +class TestBuildRoleResolverCapAuth: + @pytest.mark.asyncio + async def test_records_sync_read_failed_when_cap_auth_raises(self, journal_persisted_state): + auth_error = CapAuthError(401, "invalid cap") + inner_resolver = mock.AsyncMock(side_effect=auth_error) + with mock.patch("octobot_sync.app.create_cap_cert_role_resolver", return_value=inner_resolver): + gated_resolver = sync_app._build_role_resolver(lambda _identity: True) + with pytest.raises(CapAuthError): + await gated_resolver(mock.Mock()) + events = journal_module.read_events() + assert len(events) == 1 + event_line = events[0] + assert event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + assert event_line.attributes.collection == "user-data" + assert event_line.attributes.failure_reason == "cap_auth" + assert event_line.attributes.error_category == "CapAuthError" + + @pytest.mark.asyncio + async def test_records_sync_read_failed_when_identity_not_allowed(self, journal_persisted_state): + auth_result = mock.Mock() + auth_result.identity = "blocked-user" + inner_resolver = mock.AsyncMock(return_value=auth_result) + with mock.patch("octobot_sync.app.create_cap_cert_role_resolver", return_value=inner_resolver): + gated_resolver = sync_app._build_role_resolver(lambda identity: identity == "allowed-user") + with pytest.raises(CapAuthError, match="user not allowed"): + await gated_resolver(mock.Mock()) + events = journal_module.read_events() + assert len(events) == 1 + assert events[0].attributes.failure_reason == "cap_auth" diff --git a/packages/sync/tests/test_sync_read_failed_journal.py b/packages/sync/tests/test_sync_read_failed_journal.py new file mode 100644 index 0000000000..cad8af18ed --- /dev/null +++ b/packages/sync/tests/test_sync_read_failed_journal.py @@ -0,0 +1,117 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock +import pytest +from starfish_server.storage.base import StoreContext + +import octobot_sync.enums as sync_enums +import octobot_sync.errors as sync_errors +import octobot_sync.server as sync_server +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot.community.node_journal.events as journal_events +import octobot.community.node_journal.journal as journal_module + + +def _user_data_context() -> StoreContext: + return StoreContext( + collection=sync_enums.Collections.USER_DATA.value, + params={}, + identity="user-1", + roles=(), + action="read", + ) + + +class TestSyncReadFailureReason: + def test_maps_wallet_not_found(self): + assert sync_server._sync_read_failure_reason( + sync_errors.OctobotSyncWalletNotFoundError("missing"), + ) == "wallet_not_found" + + def test_maps_identity_missing(self): + assert sync_server._sync_read_failure_reason( + sync_errors.OctobotSyncIdentityMissingError("missing"), + ) == "identity_missing" + + def test_maps_storage_error(self): + assert sync_server._sync_read_failure_reason( + collection_errors.CollectionStorageError("broken"), + ) == "storage_error" + + def test_maps_unknown_errors_to_other(self): + assert sync_server._sync_read_failure_reason(RuntimeError("boom")) == "other" + + +class TestSyncReadFailureCollection: + def test_uses_context_collection_when_present(self): + context = StoreContext( + collection="user-accounts", + params={}, + identity="user-1", + roles=(), + action="read", + ) + assert sync_server._sync_read_failure_collection(context) == "user-accounts" + + def test_defaults_to_unknown_without_context(self): + assert sync_server._sync_read_failure_collection(None) == "unknown" + + +class TestGetData: + @pytest.mark.asyncio + async def test_records_sync_read_failed_and_reraises(self, journal_persisted_state): + expected_error = sync_errors.OctobotSyncWalletNotFoundError("wallet missing") + with mock.patch( + "octobot_sync.server.user_data_protocol.get_user_data_state", + new_callable=mock.AsyncMock, + side_effect=expected_error, + ): + with pytest.raises(sync_errors.OctobotSyncWalletNotFoundError): + await sync_server.get_data("user-data-key", _user_data_context()) + + events = journal_module.read_events() + assert len(events) == 1 + event_line = events[0] + assert event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + assert event_line.attributes.collection == sync_enums.Collections.USER_DATA.value + assert event_line.attributes.failure_reason == "wallet_not_found" + assert event_line.attributes.error_category == "OctobotSyncWalletNotFoundError" + + @pytest.mark.asyncio + async def test_successful_user_data_pull_does_not_record_sync_read_failed(self, journal_persisted_state): + user_data_state = mock.Mock() + user_data_state.to_json.return_value = "{}" + with mock.patch( + "octobot_sync.server.user_data_protocol.get_user_data_state", + new_callable=mock.AsyncMock, + return_value=user_data_state, + ), mock.patch( + "octobot_sync.server._encrypt", + return_value="encrypted", + ), mock.patch( + "octobot_sync.server.node_journal.on_user_data_pull_succeeded", + ): + result = await sync_server.get_data("user-data-key", _user_data_context()) + + assert result is not None + sync_read_failures = [ + event_line + for event_line in journal_module.read_events() + if event_line.event == journal_events.NodeJournalEvent.SYNC_READ_FAILED + ] + assert sync_read_failures == [] diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index 7f29b0493f..17b089226e 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -40,7 +40,6 @@ import octobot_commons.configuration import octobot.constants as octobot_constants -import octobot.community.activity_analysis.activity_metrics as activity_metrics import octobot.community.supabase_backend.enums as community_enums import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import @@ -92,16 +91,6 @@ def _resolve_state_file_path(recall_state: octobot_process_state_import.OctobotP ) ) -def _report_child_octobot_first_start_if_needed(init_info: dict[str, typing.Any]) -> None: - try: - if init_info.get("already_prepared"): - return - activity_metrics.ActivityMetrics.report_child_octobot_first_start() - except Exception as err: - _get_logger().exception( - err, True, f"Failed to report child OctoBot first start {err}" - ) - # --- Liveness and routing (recall state + child dump) --- @@ -1350,7 +1339,6 @@ async def _pre_compute_first_spawn( environment=child_env, hide_console_window=True, ) - _report_child_octobot_first_start_if_needed(init_info) spawn_pid = self.pid or 0 scheme = str(params.get("http_scheme") or "http").rstrip(":/") http_base_url_host = bind_host or "127.0.0.1" diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index b099a6a779..fbf6961f8c 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -1965,159 +1965,6 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp assert post.updated_exchange_account_elements is not None -class TestReportChildOctobotFirstStartIfNeeded: - def test_skips_when_layout_already_prepared(self): - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_mock: - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": True}, - ) - report_mock.assert_not_called() - - def test_reports_first_child_start_when_layout_was_created(self): - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_mock: - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": False}, - ) - report_mock.assert_called_once_with() - - def test_logs_exception_when_report_fails(self): - report_error = RuntimeError("sentry unavailable") - logger_mock = mock.Mock() - with mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - side_effect=report_error, - ), mock.patch.object( - octobot_process_ops, - "_get_logger", - return_value=logger_mock, - ): - octobot_process_ops._report_child_octobot_first_start_if_needed( - {"already_prepared": False}, - ) - logger_mock.exception.assert_called_once_with( - report_error, - True, - f"Failed to report child OctoBot first start {report_error}", - ) - - -class TestEnsureOctobotProcessPrecomputeReportsChildFirstStart: - async def test_reports_child_first_start_on_first_spawn_when_layout_is_new(self, tmp_path): - start_script = tmp_path / "start.py" - start_script.write_text("#", encoding="utf-8") - user_root = str( - tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" - ) - op = EnsureOctobotProcessOperator( - user_folder="ub", - user_id=_PROCESS_TEST_USER_ID, - profile_data=_MINIMAL_PROFILE_DATA, - last_execution_result=None, - ) - with mock.patch.object( - octobot_process_ops.os, - "getcwd", - return_value=str(tmp_path), - ), mock.patch.object( - octobot_process_ops, - "ensure_user_profile_and_layout", - new=mock.AsyncMock( - return_value={ - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - } - ), - ), mock.patch.object( - octobot_process_ops, - "_listen_port_pair_with_shared_scan_offset", - return_value=(20050, 30050), - ), mock.patch.object( - process_util, - "spawn_managed_subprocess", - ) as spawn_mock, mock.patch.object( - process_util, - "pid_is_running", - side_effect=lambda process_id: process_id == 10001, - ), mock.patch.object( - octobot_process_ops, - "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), - ), mock.patch.object( - octobot_process_ops, - "_report_child_octobot_first_start_if_needed", - ) as report_child_mock: - spawn_mock.return_value.pid = 10001 - await op.pre_compute() - report_child_mock.assert_called_once_with( - { - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - }, - ) - - async def test_reports_child_first_start_on_spawn_when_child_not_yet_live(self, tmp_path): - start_script = tmp_path / "start.py" - start_script.write_text("#", encoding="utf-8") - user_root = str( - tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" - ) - op = EnsureOctobotProcessOperator( - user_folder="ub", - user_id=_PROCESS_TEST_USER_ID, - profile_data=_MINIMAL_PROFILE_DATA, - last_execution_result=None, - ) - with mock.patch.object( - octobot_process_ops.os, - "getcwd", - return_value=str(tmp_path), - ), mock.patch.object( - octobot_process_ops, - "ensure_user_profile_and_layout", - new=mock.AsyncMock( - return_value={ - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - } - ), - ), mock.patch.object( - octobot_process_ops, - "_listen_port_pair_with_shared_scan_offset", - return_value=(20050, 30050), - ), mock.patch.object( - process_util, - "spawn_managed_subprocess", - ) as spawn_mock, mock.patch.object( - octobot_process_ops, - "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_return_none_mock), - ), mock.patch.object( - octobot_process_ops, - "_report_child_octobot_first_start_if_needed", - ) as report_child_mock: - spawn_mock.return_value.pid = 10001 - await op.pre_compute() - report_child_mock.assert_called_once_with( - { - "user_root": user_root, - "profile_id": "x", - "already_prepared": False, - }, - ) - first_le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] - assert first_le.get("init_state_ok") is False - - class TestEnsureOctobotProcessPrecomputeRecallPathWhenProcessStateLive: async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_path): start_script = tmp_path / "start.py" @@ -2155,13 +2002,9 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_return_none_mock), - ), mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_child_mock: + ): spawn_mock.return_value.pid = 10002 await op1.pre_compute() - report_child_mock.assert_not_called() first_value = op1.value assert isinstance(first_value, dict) first_le = first_value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] @@ -2185,12 +2028,8 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_live_process_bot_state_mock), - ), mock.patch.object( - octobot_process_ops.activity_metrics.ActivityMetrics, - "report_child_octobot_first_start", - ) as report_child_mock: + ): await op2.pre_compute() - report_child_mock.assert_not_called() assert isinstance(op2.value, dict) assert dsl_interpreter.ReCallingOperatorResult.__name__ in op2.value le2 = op2.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py index 292bc2239b..6a2e43ddbc 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py @@ -36,10 +36,28 @@ logs, octobots, dsl, + feedback, + journal_client_event, ) except ImportError: from api.route_provider import register_all_provider_routes # type: ignore[no-redef] - from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, accounts, debug, config, logs, octobots, dsl # type: ignore[no-redef] + from api.routes import ( # type: ignore[no-redef] + login, + nodes, + users, + tasks, + setup, + exchanges, + wallets, + accounts, + debug, + config, + logs, + octobots, + dsl, + feedback, + journal_client_event, + ) def build_api_router() -> APIRouter: @@ -58,4 +76,7 @@ def build_api_router() -> APIRouter: api_router.include_router(config.router, prefix="/config") api_router.include_router(logs.router, prefix="/logs") api_router.include_router(dsl.router, prefix="/dsl") + # See feedback / journal_client_event module docstrings for unauthenticated write endpoints. + api_router.include_router(feedback.router, prefix="/feedback") + api_router.include_router(journal_client_event.router, prefix="/journal") return api_router diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py index 06e65e2f9e..d25fcdc6b5 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py @@ -21,6 +21,7 @@ from fastapi.responses import JSONResponse import octobot_node.models +import octobot_node.enums as octobot_node_enums import octobot_node.protocol.debug as debug_protocol import octobot_node.protocol.user_actions as user_actions_protocol import octobot_node.scheduler @@ -190,7 +191,11 @@ async def execute_user_action( user_action = _parse_user_action_payload(payload) resolved_user_id = await _resolve_execution_user_id(current_user, wallet_address, user_action) try: - await user_actions_protocol.execute_user_action(user_action, resolved_user_id) + await user_actions_protocol.execute_user_action( + user_action, + resolved_user_id, + source=octobot_node_enums.UserActionSource.DEBUG_API, + ) except RuntimeError as error: if str(error) == "Scheduler is not initialized": raise HTTPException( diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py new file mode 100644 index 0000000000..246520c273 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/feedback.py @@ -0,0 +1,130 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import typing + +import pydantic +from fastapi import APIRouter + +import octobot.constants +import octobot.community.node_journal as node_journal + +try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser +except ImportError: + from api.deps import CurrentUser # type: ignore[no-redef] + +router = APIRouter(tags=["feedback"]) + + +class FeedbackUploadEnvelope(pydantic.BaseModel): + install_id: str + app_version: str + onboarding_started_at: float | None + onboarding_complete: bool + journey_summary: dict[str, typing.Any] + events: list[dict[str, typing.Any]] + uploaded: bool + ready: bool + event_count: int + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + +class FeedbackPreviewResponse(pydantic.BaseModel): + journey_summary: dict[str, typing.Any] + upload_envelope: FeedbackUploadEnvelope + + +class FeedbackUploadRequest(pydantic.BaseModel): + issue_url: str | None = None + note: str | None = None + ui_error_name: str | None = None + ui_error_route: str | None = None + + +def _feedback_events() -> list: + if not node_journal.is_journal_enabled(): + return [] + return node_journal.read_events() + + +def _build_feedback_preview() -> FeedbackPreviewResponse: + events = _feedback_events() + journey_summary = node_journal.build_journey_summary(events) + upload_envelope = node_journal.build_upload_envelope( + events, + app_version=octobot.constants.LONG_VERSION, + ) + return FeedbackPreviewResponse( + journey_summary=journey_summary.to_dict(), + upload_envelope=FeedbackUploadEnvelope(**upload_envelope.to_dict()), + ) + + +def _forward_feedback_to_remote(envelope: FeedbackUploadEnvelope) -> None: + if not node_journal.is_journal_enabled(): + envelope = envelope.model_copy(update={"events": [], "event_count": 0}) + # TODO: relay envelope to external feedback server when URL/integration is defined. + + +def _build_feedback_upload_envelope( + *, + note: str | None = None, + ui_error_name: str | None = None, + ui_error_route: str | None = None, +) -> FeedbackUploadEnvelope: + events = _feedback_events() + upload_envelope = node_journal.build_upload_envelope( + events, + app_version=octobot.constants.LONG_VERSION, + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) + return FeedbackUploadEnvelope(**upload_envelope.to_dict()) + + +@router.get("/preview", response_model=FeedbackPreviewResponse) +def get_feedback_preview(current_user: CurrentUser) -> FeedbackPreviewResponse: + return _build_feedback_preview() + + +@router.post("/upload", response_model=FeedbackUploadEnvelope) +def upload_feedback( + body: FeedbackUploadRequest | None = None, +) -> FeedbackUploadEnvelope: + note = None + ui_error_name = None + ui_error_route = None + if body is not None: + note_parts = [] + if body.note: + note_parts.append(body.note) + if body.issue_url: + note_parts.append(f"issue_url: {body.issue_url}") + if note_parts: + note = "\n".join(note_parts) + ui_error_name = body.ui_error_name + ui_error_route = body.ui_error_route + envelope = _build_feedback_upload_envelope( + note=note, + ui_error_name=ui_error_name, + ui_error_route=ui_error_route, + ) + _forward_feedback_to_remote(envelope) + return envelope diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py new file mode 100644 index 0000000000..15e3b06304 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/journal_client_event.py @@ -0,0 +1,114 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +"""Journal client-event route for the Node API. + +Security model: +- ``POST /client-event`` is intentionally unauthenticated; only allowlisted ``UI_JOURNAL_EVENTS`` are accepted. + Assumes a trusted network / same-origin UI (not a public internet-facing write API). +""" + +import json +import typing + +import pydantic +from fastapi import APIRouter, HTTPException, status + +import octobot.community.node_journal as node_journal + +router = APIRouter(tags=["journal"]) + +_MAX_CLIENT_EVENT_ATTRIBUTES_BYTES = 4096 + + +class JournalClientEventRequest(pydantic.BaseModel): + event: str + client_instance_id: str | None = None + attributes: dict[str, typing.Any] | None = None + + +class JournalClientEventResponse(pydantic.BaseModel): + event: str + timestamp: float + session_id: str + install_id: str + app_version: str + distribution: str + onboarding_complete: bool + attributes: dict[str, typing.Any] + recorded: bool = True + + +def _validate_client_event(event_name: str) -> node_journal.NodeJournalEvent: + try: + parsed_event = node_journal.NodeJournalEvent(event_name) + except ValueError as error: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown journal event: {event_name}", + ) from error + if parsed_event not in node_journal.UI_JOURNAL_EVENTS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Event not allowed: {event_name}", + ) + return parsed_event + + +def _build_client_event_attributes(body: JournalClientEventRequest) -> dict[str, typing.Any]: + attributes = dict(body.attributes or {}) + if body.client_instance_id is not None: + attributes["client_instance_id"] = body.client_instance_id + if len(json.dumps(attributes, default=str)) > _MAX_CLIENT_EVENT_ATTRIBUTES_BYTES: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client event attributes payload is too large", + ) + return attributes + + +@router.post("/client-event", response_model=JournalClientEventResponse) +def record_journal_client_event(body: JournalClientEventRequest) -> JournalClientEventResponse: + parsed_event = _validate_client_event(body.event) + attributes = _build_client_event_attributes(body) + event_line = node_journal.record(parsed_event, attributes=attributes) + response_attributes = dict(attributes) + if event_line.attributes.raw_event_name is not None: + response_attributes["raw_event_name"] = event_line.attributes.raw_event_name + event_value = event_line.event.value + if event_line.recorded is False: + return JournalClientEventResponse( + event=event_value, + timestamp=event_line.timestamp, + session_id=event_line.session_id, + install_id=event_line.install_id, + app_version=event_line.app_version, + distribution=event_line.distribution, + onboarding_complete=event_line.onboarding_complete, + attributes=response_attributes, + recorded=False, + ) + return JournalClientEventResponse( + event=event_value, + timestamp=event_line.timestamp, + session_id=event_line.session_id, + install_id=event_line.install_id, + app_version=event_line.app_version, + distribution=event_line.distribution, + onboarding_complete=event_line.onboarding_complete, + attributes=response_attributes, + recorded=True, + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py index facaa6bfd1..85e5b858b9 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py @@ -23,6 +23,9 @@ import octobot_node.config as node_config import octobot.community.authentication as community_auth import octobot.community.wallet_backend as wallet_backend +import octobot.community.node_journal as node_journal +import octobot.community.node_journal.enums as journal_enums +import octobot.community.node_journal.recording_context as journal_recording_context try: from api.deps import CurrentUser, security_basic # type: ignore[no-redef] @@ -83,16 +86,27 @@ def get_vpn_network_address() -> VPNNetworkAddress: @router.post("/setup/init", response_model=SetupResult) def init_setup(body: SetupInit) -> SetupResult: auth = community_auth.CommunityAuthentication.instance() + setup_method = journal_enums.WalletSetupMethod.IMPORT if body.private_key else journal_enums.WalletSetupMethod.CREATE if auth is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_503_SERVICE_UNAVAILABLE, + failure_reason=journal_enums.WalletSetupFailureReason.SERVICE_UNAVAILABLE, + setup_method=setup_method, detail="Service not initialized", + error_message="Service not initialized", ) if auth.list_wallets(): - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_409_CONFLICT, + failure_reason=journal_enums.WalletSetupFailureReason.ALREADY_CONFIGURED, + setup_method=setup_method, detail="Node is already configured", + error_message="Node is already configured", ) + node_journal.record_wallet_setup_attempt( + node_type=body.node_type, + setup_method=setup_method, + ) try: if body.private_key: wallet = auth.import_wallet( @@ -108,17 +122,23 @@ def init_setup(body: SetupInit) -> SetupResult: is_admin=True, ) except (wallet_backend.WalletAlreadyExistsError, wallet_backend.AdminWalletAlreadyExistsError) as err: - # A concurrent request already configured the node — surface 409. - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_409_CONFLICT, + failure_reason=journal_enums.WalletSetupFailureReason.CONCURRENT_RACE, + setup_method=setup_method, detail=str(err), - ) from err + error=err, + ) except wallet_backend.WalletError as err: - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + journal_recording_context.raise_wallet_setup_http_error( + http_status=status.HTTP_422_UNPROCESSABLE_ENTITY, + failure_reason=journal_enums.WalletSetupFailureReason.WALLET_ERROR, + setup_method=setup_method, detail=str(err), - ) from err + error=err, + ) node_config.settings.IS_MASTER_MODE = body.node_type == "master" + node_journal.record_wallet_setup_succeeded() return SetupResult(address=wallet.address) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py index 5952f8f90c..bd97f249cb 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py @@ -20,6 +20,11 @@ from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE +_WALLET_SETUP_FAILED_PATCH = ( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup." + "journal_recording_context.journal_recording.record_wallet_setup_failed" +) + _INIT_BODY = { "passphrase": "strongpass123", "node_type": "standalone", @@ -58,6 +63,8 @@ def test_setup_init_success(client): with mock.patch( "octobot.community.authentication.CommunityAuthentication.instance", return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", ): with mock.patch("octobot_node.config.settings"): resp = client.post("/api/v1/setup/init", json=_INIT_BODY) @@ -68,6 +75,37 @@ def test_setup_init_success(client): ) +def test_setup_init_emits_wallet_setup_succeeded(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [] + auth.create_wallet.return_value = mock.MagicMock(address=ADMIN_ADDRESS) + metrics_config = mock.MagicMock() + auth.config = metrics_config + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", + ) as record_wallet_setup_succeeded_mock, mock.patch("octobot_node.config.settings"): + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 200 + record_wallet_setup_succeeded_mock.assert_called_once_with() + + +def test_setup_init_skips_wallet_milestone_on_409(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [{"address": ADMIN_ADDRESS, "is_admin": True}] + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", + ) as record_wallet_setup_succeeded_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 409 + record_wallet_setup_succeeded_mock.assert_not_called() + + def test_setup_init_with_private_key(client): pk = "a" * 64 auth = mock.MagicMock() @@ -76,6 +114,8 @@ def test_setup_init_with_private_key(client): with mock.patch( "octobot.community.authentication.CommunityAuthentication.instance", return_value=auth, + ), mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.node_journal.record_wallet_setup_succeeded", ): with mock.patch("octobot_node.config.settings"): resp = client.post( @@ -99,6 +139,36 @@ def test_setup_init_already_configured_returns_409(client): assert resp.status_code == 409 +def test_setup_init_records_wallet_setup_failed_on_409(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [{"address": ADMIN_ADDRESS, "is_admin": True}] + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 409 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 409 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "already_configured" + + +def test_setup_init_records_wallet_setup_failed_on_503(client): + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=None, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock: + resp = client.post("/api/v1/setup/init", json=_INIT_BODY) + assert resp.status_code == 503 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 503 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "service_unavailable" + + def test_setup_init_invalid_passphrase_returns_422(client): auth = mock.MagicMock() auth.list_wallets.return_value = [] @@ -115,6 +185,26 @@ def test_setup_init_invalid_passphrase_returns_422(client): assert resp.status_code == 422 +def test_setup_init_records_wallet_setup_failed_on_422(client): + auth = mock.MagicMock() + auth.list_wallets.return_value = [] + auth.create_wallet.side_effect = wallet_backend.WalletError("Passphrase must be at least 8 characters") + with mock.patch( + "octobot.community.authentication.CommunityAuthentication.instance", + return_value=auth, + ), mock.patch( + _WALLET_SETUP_FAILED_PATCH, + ) as record_wallet_setup_failed_mock, mock.patch("octobot_node.config.settings"): + resp = client.post( + "/api/v1/setup/init", + json={**_INIT_BODY, "passphrase": "short"}, + ) + assert resp.status_code == 422 + record_wallet_setup_failed_mock.assert_called_once() + assert record_wallet_setup_failed_mock.call_args.kwargs["http_status"] == 422 + assert record_wallet_setup_failed_mock.call_args.kwargs["failure_reason"] == "wallet_error" + + def test_wallet_export_success(admin_client, mock_auth): mock_auth.decrypt_wallet_entry_by_address.return_value = mock.MagicMock( address=ADMIN_ADDRESS, diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx new file mode 100644 index 0000000000..30a28b4eff --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/__tests__/bootstrap-app.test.tsx @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + createRouter: vi.fn(() => ({ id: "router-mock" })), + isWebCryptoAvailable: vi.fn(() => true), + probeAuthState: vi.fn(async () => "ok" as const), + reportInsecureContext: vi.fn(), + render: vi.fn(), +})) + +vi.mock("react-dom/client", () => ({ + createRoot: () => ({ + render: mocks.render, + }), +})) + +vi.mock("@tanstack/react-router", () => ({ + createRouter: mocks.createRouter, + RouterProvider: () => null, +})) + +vi.mock("@/routeTree.gen", () => ({ + routeTree: {}, +})) + +vi.mock("@/lib/secure-context", () => ({ + isWebCryptoAvailable: mocks.isWebCryptoAvailable, +})) + +vi.mock("@/lib/auth-state-probe", () => ({ + probeAuthState: mocks.probeAuthState, +})) + +vi.mock("@/lib/shell-error-reporting", () => ({ + reportInsecureContext: mocks.reportInsecureContext, + reportAuthStateBroken: vi.fn(), + reportShellFatalError: vi.fn(), +})) + +vi.mock("@/lib/device-key", () => ({ + loadPassword: vi.fn(async () => null), +})) + +import { bootstrapApp } from "@/bootstrap-app" + +describe("bootstrapApp insecure context", () => { + beforeEach(() => { + mocks.createRouter.mockClear() + mocks.isWebCryptoAvailable.mockReturnValue(true) + mocks.probeAuthState.mockResolvedValue("ok") + mocks.reportInsecureContext.mockClear() + mocks.render.mockClear() + vi.stubGlobal("document", { + getElementById: vi.fn(() => ({ id: "root" })), + }) + vi.stubGlobal("window", { + isSecureContext: false, + location: { hostname: "192.168.0.1", pathname: "/app", port: "8000" }, + }) + }) + + it("renders recovery and skips router when web crypto is unavailable", async () => { + mocks.isWebCryptoAvailable.mockReturnValue(false) + + await bootstrapApp() + + expect(mocks.reportInsecureContext).toHaveBeenCalledOnce() + expect(mocks.render).toHaveBeenCalledOnce() + expect(mocks.createRouter).not.toHaveBeenCalled() + expect(mocks.probeAuthState).not.toHaveBeenCalled() + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx new file mode 100644 index 0000000000..df164dd05b --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/bootstrap-app.tsx @@ -0,0 +1,150 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query" +import { createRouter, RouterProvider } from "@tanstack/react-router" +import { StrictMode } from "react" +import { createRoot } from "react-dom/client" +import { ErrorBoundary } from "react-error-boundary" +import { OpenAPI } from "@/client" +import { RecoveryScreen } from "@/components/Common/RecoveryScreen" +import { ThemeProvider } from "@/components/theme-provider" +import { Toaster } from "@/components/ui/sonner" +import { clearAuth } from "@/hooks/useAuth" +import { probeAuthState } from "@/lib/auth-state-probe" +import { shouldRedirectToLoginOn401 } from "@/lib/open-api-401-login-redirect" +import { loadPassword } from "@/lib/device-key" +import type { RecoveryFailureKind } from "@/components/Common/RecoveryScreen" +import { isWebCryptoAvailable } from "@/lib/secure-context" +import { + reportAuthStateBroken, + reportInsecureContext, + reportShellFatalError, +} from "@/lib/shell-error-reporting" +import { routeTree } from "@/routeTree.gen" + +// Wire generated API client credentials and session recovery before any route loads. +export function configureOpenApi(): void { + OpenAPI.BASE = + import.meta.env.NODE_API_URL || + (import.meta.env.DEV ? "http://localhost:8000" : "") + OpenAPI.USERNAME = async () => { + return localStorage.getItem("auth_username") || "" + } + OpenAPI.PASSWORD = async () => { + return (await loadPassword()) ?? "" + } + + let isRedirectingOnAuthFailure = false + // On 401 for a protected page, clear local session and hard-navigate to login so the UI + // never keeps calling APIs with a stale username and no usable password. + OpenAPI.interceptors.response.use((response) => { + if ( + shouldRedirectToLoginOn401(response, { + pathname: window.location.pathname, + isRedirectingOnAuthFailure, + }) + ) { + isRedirectingOnAuthFailure = true + void clearAuth().finally(() => { + window.location.href = "/app/login" + }) + } + return response + }) +} + +function createAppRouter() { + return createRouter({ + routeTree, + basepath: "/app", + }) +} + +type RecoveryView = + | { mode: "recovery"; failureKind: RecoveryFailureKind } + | { mode: "app" } + +async function resolveStartupView(): Promise { + // probeAuthState: skipped during setup; ok when logged out or password present; + // broken when auth_username exists but IndexedDB password is missing/unreadable. + const authState = await probeAuthState() + if (authState === "broken") { + reportAuthStateBroken() + return { mode: "recovery", failureKind: "auth_broken" } + } + return { mode: "app" } +} + +function renderRecovery( + rootElement: HTMLElement, + failureKind: RecoveryFailureKind, +): void { + const queryClient = new QueryClient() + + createRoot(rootElement).render( + + + + + + + + , + ) +} + +// Shown when a render error escapes the main app tree (ErrorBoundary fallback). +export function ShellErrorFallback() { + return +} + +export function handleShellRenderError(error: unknown): void { + reportShellFatalError( + error instanceof Error ? error : new Error(String(error)), + ) +} + +function renderApp(rootElement: HTMLElement): void { + const queryClient = new QueryClient() + const router = createAppRouter() + + createRoot(rootElement).render( + + + + + + + + + + , + ) +} + +// Startup: configure API → mount root → require Web Crypto → auth probe → app or RecoveryScreen. +export async function bootstrapApp(): Promise { + configureOpenApi() + const rootElement = document.getElementById("root") + if (!rootElement) { + throw new Error("Root element not found") + } + + if (!isWebCryptoAvailable()) { + // Non-secure context (e.g. HTTP remote UI): crypto.subtle unavailable → insecure_context recovery. + reportInsecureContext({ + isSecureContext: window.isSecureContext, + hostname: window.location.hostname, + }) + renderRecovery(rootElement, "insecure_context") + return + } + + const startupView = await resolveStartupView() + if (startupView.mode === "recovery") { + renderRecovery(rootElement, startupView.failureKind) + return + } + renderApp(rootElement) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx index 84b39c6e18..fb3b6bfbc4 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/AppHeader.tsx @@ -2,6 +2,7 @@ import { Link } from "@tanstack/react-router" import { Plus } from "lucide-react" import { Logo } from "@/components/Common/Logo" +import { ShareFeedbackButton } from "@/components/Common/ShareFeedbackButton" import UserMenu from "@/components/Common/UserMenu" import { Button } from "@/components/ui/button" @@ -11,6 +12,11 @@ export function AppHeader() {
+ - +
+ + + + +
) } diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx new file mode 100644 index 0000000000..c2f3f992c1 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextGuidance.tsx @@ -0,0 +1,66 @@ +import { buildSameComputerLoopbackUrl } from "@/lib/secure-context" +import { + INSECURE_CONTEXT_OPTION_LOCAL_SUBLABEL, + INSECURE_CONTEXT_OPTION_LOCAL_TITLE, + INSECURE_CONTEXT_OPTION_REMOTE_TITLE, + INSECURE_CONTEXT_REMOTE_LEAD, + INSECURE_CONTEXT_SAME_COMPUTER_LEAD, + INSECURE_CONTEXT_WHY_ADDRESS_AFTER, + INSECURE_CONTEXT_WHY_ADDRESS_BEFORE, + INSECURE_CONTEXT_WHY_LEAD, + TAILSCALE_REMOTE_ACCESS_BUTTON_LABEL, +} from "@/lib/ui-recovery-constants" +import { Button } from "@/components/ui/button" + +type InsecureContextGuidanceProps = { + onSetUpRemoteAccessClick: () => void +} + +export function InsecureContextGuidance({ + onSetUpRemoteAccessClick, +}: InsecureContextGuidanceProps) { + const pageHref = + typeof window !== "undefined" ? window.location.href : "" + const loopbackUrl = + typeof window !== "undefined" + ? buildSameComputerLoopbackUrl(window.location.href) + : "http://127.0.0.1:8000/app" + + return ( +
+
+

{INSECURE_CONTEXT_WHY_LEAD}

+ {pageHref !== "" ? ( +

+ {INSECURE_CONTEXT_WHY_ADDRESS_BEFORE} + {pageHref} + {INSECURE_CONTEXT_WHY_ADDRESS_AFTER} +

+ ) : null} +
+
+

{INSECURE_CONTEXT_OPTION_LOCAL_TITLE}

+

{INSECURE_CONTEXT_OPTION_LOCAL_SUBLABEL}

+

{INSECURE_CONTEXT_SAME_COMPUTER_LEAD}

+

+ + {loopbackUrl} + +

+
+
+

{INSECURE_CONTEXT_OPTION_REMOTE_TITLE}

+

{INSECURE_CONTEXT_REMOTE_LEAD}

+ +
+
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx deleted file mode 100644 index 0e954c740d..0000000000 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/InsecureContextNotice.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { getLoopbackUrl } from "@/lib/secure-context" - -const InsecureContextNotice = () => { - const loopbackUrl = getLoopbackUrl(window.location.href) - - return ( -
-
- - 🔒 - -

Secure context required

-

- This browser origin does not support the cryptography the node needs. - Open the node UI from a secure address instead. -

- {loopbackUrl ? ( -

- - {loopbackUrl} - -

- ) : ( -

- On this machine, open{" "} - - http://127.0.0.1:{window.location.port || "8000"}/app - - . To reach the node from another device, serve it over HTTPS. -

- )} -
-
- ) -} - -export default InsecureContextNotice diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx new file mode 100644 index 0000000000..6e11ee9c5f --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/MagicDnsRemoteUrlField.tsx @@ -0,0 +1,77 @@ +import { SelectableTextBlock } from "@/components/Common/SelectableTextBlock" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { + buildHttpsNodeUrlFromHostname, + parseMagicDnsHostnameInput, +} from "@/lib/secure-context" +import { + MAGIC_DNS_INPUT_LABEL, + MAGIC_DNS_INPUT_PLACEHOLDER, + MAGIC_DNS_OPEN_IN_BROWSER_LABEL, + MAGIC_DNS_REMOTE_URL_LABEL, + TAILSCALE_AVOID_IP_HTTPS_WARNING, +} from "@/lib/ui-recovery-constants" + +type MagicDnsRemoteUrlFieldProps = { + pageHref: string + value: string + onValueChange: (value: string) => void +} + +const REMOTE_URL_ARIA_LABEL = "Remote HTTPS URL" + +export function MagicDnsRemoteUrlField({ + pageHref, + value, + onValueChange, +}: MagicDnsRemoteUrlFieldProps) { + const trimmed = value.trim() + const parsed = + trimmed === "" ? null : parseMagicDnsHostnameInput(value) + const remoteUrl = + parsed?.ok === true + ? buildHttpsNodeUrlFromHostname(parsed.hostname, pageHref) + : null + const errorMessage = + trimmed !== "" && parsed?.ok === false ? parsed.message : null + + return ( +
+
+ + onValueChange(event.target.value)} + autoComplete="off" + /> + {errorMessage ? ( +

{errorMessage}

+ ) : null} +
+ {remoteUrl ? ( +
+ {MAGIC_DNS_REMOTE_URL_LABEL} + + + {MAGIC_DNS_OPEN_IN_BROWSER_LABEL} + +

+ {TAILSCALE_AVOID_IP_HTTPS_WARNING} +

+
+ ) : null} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx new file mode 100644 index 0000000000..cd3b25ea38 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/RecoveryScreen.tsx @@ -0,0 +1,97 @@ +import { useState } from "react" + +import { InsecureContextGuidance } from "@/components/Common/InsecureContextGuidance" +import { ShareFeedbackButton } from "@/components/Common/ShareFeedbackButton" +import { TailscaleRemoteAccessDialog } from "@/components/Common/TailscaleRemoteAccessDialog" +import { resetClientStorage } from "@/lib/client-storage-reset" +import { + INSECURE_CONTEXT_HEADING, + RECOVERY_EXPLANATION, + RESET_CONFIRM_MESSAGE, +} from "@/lib/ui-recovery-constants" +import { Button } from "@/components/ui/button" + +export type RecoveryFailureKind = + | "boot_failed" + | "auth_broken" + | "fatal_render" + | "insecure_context" + +const FAILURE_HEADINGS: Record = { + boot_failed: "OctoBot Node failed to start", + auth_broken: "Sign-in data is inconsistent", + fatal_render: "OctoBot Node encountered a fatal error", + insecure_context: INSECURE_CONTEXT_HEADING, +} + +type RecoveryScreenProps = { + failureKind: RecoveryFailureKind +} + +export function confirmAndResetLocalBrowserData( + confirmFn: () => boolean = () => window.confirm(RESET_CONFIRM_MESSAGE), +): void { + if (!confirmFn()) { + return + } + void resetClientStorage("manual_recovery") +} + +export function RecoveryScreen({ failureKind }: RecoveryScreenProps) { + const [tailscaleDialogOpen, setTailscaleDialogOpen] = useState(false) + + const handleResetClick = () => { + confirmAndResetLocalBrowserData() + } + + const handleReloadClick = () => { + window.location.reload() + } + + const isInsecureContext = failureKind === "insecure_context" + const explanation = isInsecureContext ? null : RECOVERY_EXPLANATION + const feedbackContext = { source: "recovery" as const, failureKind } + + return ( +
+
+

{FAILURE_HEADINGS[failureKind]}

+ {isInsecureContext ? ( + <> + setTailscaleDialogOpen(true)} + /> +
+ +
+ + ) : ( +

{explanation}

+ )} + {!isInsecureContext ? ( +
+ + + +
+ ) : null} +
+ {isInsecureContext ? ( + + ) : null} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx new file mode 100644 index 0000000000..85f625412e --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/SelectableTextBlock.tsx @@ -0,0 +1,60 @@ +import type { FocusEvent, MouseEvent } from "react" + +import { cn } from "@/lib/utils" + +type SelectableTextBlockProps = { + value: string + ariaLabel: string + multiline?: boolean +} + +const selectableControlClassName = cn( + "w-full cursor-text border-0 bg-transparent p-0 font-mono text-xs text-foreground", + "break-all outline-none focus-visible:ring-0", +) + +function selectAllInControl( + event: FocusEvent, +): void { + event.currentTarget.select() +} + +function selectAllInControlOnClick( + event: MouseEvent, +): void { + event.currentTarget.select() +} + +export function SelectableTextBlock({ + value, + ariaLabel, + multiline = false, +}: SelectableTextBlockProps) { + const textareaRows = Math.max(2, value.split("\n").length) + + return ( +
+ {multiline ? ( +