migrate command (#61)

src/demo_hic_et_nunc/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_hic_et_nunc
 
 database:

src/demo_quipuswap/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_quipuswap
 
 database:

src/demo_registrydao/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_registrydao
 
 database:

src/demo_tezos_domains/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tezos_domains
 
 database:

src/demo_tezos_domains_big_map/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tezos_domains_big_map
 
 database:

src/demo_tzbtc/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tzbtc
 
 database:

src/demo_tzcolors/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tzcolors
 
 database:

src/dipdup/__init__.py

@@ -1 +1,2 @@
 __version__ = '0.4.3'
+__spec_version__ = '1.0'

src/dipdup/cli.py

@@ -1,21 +1,50 @@
 import asyncio
+import fileinput
 import logging
 import os
 from dataclasses import dataclass
 from functools import wraps
 from os.path import dirname, join
-from typing import List
+from typing import List, NoReturn
 
 import click
 from fcache.cache import FileCache  # type: ignore
 
-from dipdup import __version__
+from dipdup import __spec_version__, __version__
 from dipdup.config import DipDupConfig, LoggingConfig
 from dipdup.dipdup import DipDup
-from dipdup.exceptions import HandlerImportError
 
 _logger = logging.getLogger(__name__)
 
+spec_version_to_version = {
+    '0.1': 'dipdup <0.4.3',
+    '1.0': 'dipdup ^1.0.0',
+}
+
+migration_required_message = """
+
+Migration required!
+
+project spec version: %s (%s)
+current spec version: %s (%s)
+
+  1. Run `dipdup migrate`
+  2. Review and commit changes
+
+See https://baking-bad.org/blog/ for additional release information.
+"""
+
+
+def migration_required(from_: str, to: str) -> NoReturn:
+    _logger.warning(
+        migration_required_message,
+        from_,
+        spec_version_to_version[from_],
+        to,
+        spec_version_to_version[to],
+    )
+    quit()
+
 
 def click_async(fn):
     @wraps(fn)
@@ -27,6 +56,7 @@ def wrapper(*args, **kwargs):
 
 @dataclass
 class CLIContext:
+    config_paths: List[str]
     config: DipDupConfig
     logging_config: LoggingConfig
 
@@ -45,8 +75,13 @@ async def cli(ctx, config: List[str], logging_config: str):
         path = join(dirname(__file__), 'configs', logging_config)
         _logging_config = LoggingConfig.load(path)
     _logging_config.apply()
+
     _config = DipDupConfig.load(config)
+    if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
+        migration_required(_config.spec_version, __spec_version__)
+
     ctx.obj = CLIContext(
+        config_paths=config,
         config=_config,
         logging_config=_logging_config,
     )
@@ -59,12 +94,7 @@ async def cli(ctx, config: List[str], logging_config: str):
 @click_async
 async def run(ctx, reindex: bool, oneshot: bool) -> None:
     config: DipDupConfig = ctx.obj.config
-
-    try:
-        config.initialize()
-    except HandlerImportError:
-        await DipDup(config).migrate()
-
+    config.initialize()
     dipdup = DipDup(config)
     await dipdup.run(reindex, oneshot)
 
@@ -79,6 +109,22 @@ async def init(ctx):
     await dipdup.init()
 
 
+@cli.command(help='Migrate project to the new spec version')
+@click.pass_context
+@click_async
+async def migrate(ctx):
+    config: DipDupConfig = ctx.obj.config
+    config.pre_initialize()
+    await DipDup(config).migrate()
+
+    for config_path in ctx.obj.config_paths:
+        for line in fileinput.input(config_path, inplace=True):
+            if 'spec_version' in line:
+                print(f'spec_version: {__spec_version__}')
+            else:
+                print(line.rstrip())
+
+
 @cli.command(help='Clear development request cache')
 @click.pass_context
 @click_async

src/dipdup/codegen.py

@@ -249,20 +249,14 @@ async def generate_default_handlers(self, recreate=False) -> None:
         self._logger.info('Generating handler `%s`', CONFIGURE_HANDLER)
         handler_code = configure_template.render()
         handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py')
-        if exists(handler_path) and recreate:
-            old_handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py.bak')
-            os.rename(handler_path, old_handler_path)
-        if not exists(handler_path):
+        if not exists(handler_path) or recreate:
             with open(handler_path, 'w') as file:
                 file.write(handler_code)
 
         self._logger.info('Generating handler `%s`', ROLLBACK_HANDLER)
         handler_code = rollback_template.render()
         handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py')
-        if exists(handler_path) and recreate:
-            old_handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py.bak')
-            os.rename(handler_path, old_handler_path)
-        if not exists(handler_path):
+        if not exists(handler_path) or recreate:
             with open(handler_path, 'w') as file:
                 file.write(handler_code)
 
@@ -342,7 +336,6 @@ async def _get_schema(
         return self._schemas[datasource_config][address]
 
     async def migrate_user_handlers_to_v1(self) -> None:
-        # TODO: Save backups
         remove_lines = [
             'from dipdup.models import',
             'from dipdup.context import',
@@ -356,8 +349,8 @@ async def migrate_user_handlers_to_v1(self) -> None:
             'TransactionContext': 'Transaction',
             'OriginationContext': 'Origination',
             'BigMapContext': 'BigMapDiff',
-            'HandlerContext': 'HandlerContext',
-            'HandlerContext': 'HandlerContext',
+            'OperationHandlerContext': 'HandlerContext',
+            'BigMapHandlerContext': 'HandlerContext',
         }
         handlers_path = join(self._config.package_path, 'handlers')
 
@@ -366,10 +359,8 @@ async def migrate_user_handlers_to_v1(self) -> None:
                 if filename == '__init__.py' or not filename.endswith('.py'):
                     continue
                 path = join(root, filename)
-                bak_path = path + '.bak'
-                os.rename(path, bak_path)
                 newfile = copy(add_lines)
-                with open(bak_path) as file:
+                with open(path) as file:
                     for line in file.read().split('\n'):
                         # Skip existing models imports
                         if any(map(lambda l: l in line, remove_lines)):

src/dipdup/config.py

@@ -623,12 +623,6 @@ def valid_url(cls, v):
         return v
 
 
-@dataclass
-class ConfigurationConfig:
-    interval: int = 60
-    args: Dict[str, Any] = Field(default_factory=dict)
-
-
 @dataclass
 class DipDupConfig:
     """Main dapp config
@@ -652,7 +646,6 @@ class DipDupConfig:
     templates: Optional[Dict[str, IndexConfigTemplateT]] = None
     database: Union[SqliteDatabaseConfig, MySQLDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite')
     hasura: Optional[HasuraConfig] = None
-    configuration: Optional[ConfigurationConfig] = None
 
     def __post_init_post_parse__(self):
         self._callback_patterns: Dict[str, List[Sequence[HandlerPatternConfigT]]] = defaultdict(list)
@@ -843,15 +836,9 @@ def load(
 
     def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None:
         _logger.info('Registering handler callback `%s`', handler_config.callback)
-        try:
-            handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}')
-            callback_fn = getattr(handler_module, handler_config.callback)
-            handler_config.callback_fn = callback_fn
-        except ImportError as e:
-            if 'Context' in str(e):
-                _logger.warning('Found broken imports, attemping to fix them')
-                raise HandlerImportError from e
-            raise
+        handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}')
+        callback_fn = getattr(handler_module, handler_config.callback)
+        handler_config.callback_fn = callback_fn
 
     def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
         if index_name in self._initialized:

src/dipdup/dipdup.py

@@ -46,7 +46,7 @@ def __init__(self, ctx: HandlerContext) -> None:
     async def add_index(self, index_config: IndexConfigTemplateT) -> None:
         if index_config.name in self._indexes:
             return
-        self._logger.info('Adding index `%s` to dispatcher')
+        self._logger.info('Adding index `%s` to dispatcher', index_config.name)
         if isinstance(index_config, OperationIndexConfig):
             datasource_name = cast(TzktDatasourceConfig, index_config.datasource).name
             datasource = self._ctx.datasources[datasource_name]

tests/integration_tests/hic_et_nunc.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_hic_et_nunc
 
 database:

tests/integration_tests/quipuswap.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_quipuswap
 
 database:

tests/integration_tests/registrydao.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_registrydao
 
 database:

tests/integration_tests/tezos_domains.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tezos_domains
 
 database:

tests/integration_tests/tezos_domains_big_map.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tezos_domains_big_map
 
 database:

tests/integration_tests/tzcolors.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_tzcolors
 
 database:

tests/test_dipdup/dipdup.yml

@@ -1,4 +1,4 @@
-spec_version: 0.1
+spec_version: 1.0
 package: demo_hic_et_nunc
 
 database: