Version 3.0.0-beta3

.n6-version

@@ -1 +1 @@
-3.0.0b2
+3.0.0b3

N6BrokerAuthApi/n6brokerauthapi/__init__.py

@@ -1,5 +1,5 @@
-# Copyright (c) 2013-2019 NASK. All rights reserved.
-
+# Copyright (c) 2013-2021 NASK. All rights reserved.
+#TODO: Module modernized to Python 3, but no changes detected, comment to be deleted after MR
 """
 This package provides a REST API implementation intended to cooperate
 with `rabbitmq-auth-backend-http` -- the RabbitMQ AMQP message broker's

N6BrokerAuthApi/n6brokerauthapi/auth_base.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2019 NASK. All rights reserved.
+# Copyright (c) 2013-2021 NASK. All rights reserved.
 
 import sys
 import threading
@@ -17,7 +17,7 @@
 LOGGER = get_logger(__name__)
 
 
-class BaseBrokerAuthManagerMaker(object):
+class BaseBrokerAuthManagerMaker:
 
     def __init__(self, settings):
         self._db_connector = SQLAuthDBConnector(settings=settings)
@@ -47,7 +47,7 @@ def get_manager_factory_kwargs(self, validated_view_params):
                     params=validated_view_params)
 
 
-class BaseBrokerAuthManager(object):
+class BaseBrokerAuthManager:
 
     def __init__(self,
                  db_connector,
@@ -114,9 +114,9 @@ def client_type(self):
         assert self.client_obj is not None
         if isinstance(self.client_obj, models.User):
             return 'user'
-        elif isinstance(self.client_obj, models.Component):
+        if isinstance(self.client_obj, models.Component):
             return 'component'
-        raise TypeError('the client object {!r} is an instance of '
+        raise TypeError('the client object {!a} is an instance of '
                         'a wrong class'.format(self.client_obj))
 
     @property
@@ -134,7 +134,7 @@ def _get_admins_group(self):
             return self.db_session.query(models.SystemGroup).filter(
                 models.SystemGroup.name == ADMINS_SYSTEM_GROUP_NAME).one()
         except NoResultFound:
-            LOGGER.error('System group %r not found in auth db!', ADMINS_SYSTEM_GROUP_NAME)
+            LOGGER.error('System group %a not found in auth db!', ADMINS_SYSTEM_GROUP_NAME)
             return None
 
     #

N6BrokerAuthApi/n6brokerauthapi/auth_stream_api.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2019 NASK. All rights reserved.
+# Copyright (c) 2013-2021 NASK. All rights reserved.
 
 import re
 
@@ -28,7 +28,7 @@ class StreamApiBrokerAuthManagerMaker(ConfigMixin, BaseBrokerAuthManagerMaker):
     """
 
     def __init__(self, settings):
-        super(StreamApiBrokerAuthManagerMaker, self).__init__(settings=settings)
+        super().__init__(settings=settings)
         self._config = self.get_config_section(settings)
         self._thread_local = ThreadLocalNamespace(attr_factories={
             'autogenerated_queue_matcher': self._make_autogenerated_queue_matcher,
@@ -46,7 +46,7 @@ def get_manager_factory(self, params):
         return StreamApiBrokerAuthManager
 
     def get_manager_factory_kwargs(self, params):
-        base = super(StreamApiBrokerAuthManagerMaker, self).get_manager_factory_kwargs(params)
+        base = super().get_manager_factory_kwargs(params)
         return dict(base,
                     push_exchange_name=self._config['push_exchange_name'] or None,
                     privileged_component_logins=self._config['privileged_component_logins'],
@@ -63,23 +63,23 @@ def __init__(self,
         self._push_exchange_name = push_exchange_name
         self._privileged_component_logins = privileged_component_logins
         self._autogenerated_queue_matcher = autogenerated_queue_matcher
-        super(StreamApiBrokerAuthManager, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
 
     EXPLICITLY_ILLEGAL_USERNAMES = ('', 'guest')
 
     def should_try_to_verify_client(self):
         if self.broker_username in self.EXPLICITLY_ILLEGAL_USERNAMES:
             LOGGER.error(
-                "The '%s' username is explicitly considered illegal!",
+                "The '%a' username is explicitly considered illegal!",
                 ascii_str(self.broker_username))
             return False
         if self.password is not None:
             LOGGER.error(
-                "Authentication by password is not supported - cannot authenticate '%s'!",
+                "Authentication by password is not supported - cannot authenticate '%a'!",
                 ascii_str(self.broker_username))
             return False
-        return super(StreamApiBrokerAuthManager, self).should_try_to_verify_client()
+        return super().should_try_to_verify_client()
 
     def verify_and_get_user_obj(self):
         user_obj = self._from_db(models.User, 'login', self.broker_username)

N6BrokerAuthApi/n6brokerauthapi/tests/test_views_with_auth_stream_api.py

@@ -1,9 +1,9 @@
-# Copyright (c) 2013-2019 NASK. All rights reserved.
+# Copyright (c) 2013-2021 NASK. All rights reserved.
 
 import itertools
 import unittest
 
-from mock import (
+from unittest.mock import (
     MagicMock,
     call,
     patch,
@@ -167,22 +167,22 @@ def assertConnectorUsedOnlyAfterEnsuredClean(self):
 
 
 # noinspection PyUnresolvedReferences
-class _AssertResponseMixin(object):
+class _AssertResponseMixin:
 
     def assertAllow(self, resp):
-        self.assertIn(resp.body, ['allow', 'allow administrator'])
+        self.assertIn(resp.body, [b'allow', b'allow administrator'])
         self.assertEqual(resp.status_code, 200)
 
     def assertDeny(self, resp):
-        self.assertEqual(resp.body, 'deny')
+        self.assertEqual(resp.body, b'deny')
         self.assertEqual(resp.status_code, 200)
 
     def assertAdministratorTagPresent(self, resp):
-        self.assertIn('administrator', resp.body.split())
+        self.assertIn(b'administrator', resp.body.split())
         self.assertEqual(resp.status_code, 200)
 
     def assertNoAdministratorTag(self, resp):
-        self.assertNotIn('administrator', resp.body.split())
+        self.assertNotIn(b'administrator', resp.body.split())
         self.assertEqual(resp.status_code, 200)
 
 
@@ -210,15 +210,15 @@ def basic_allow_params(cls):
     @paramseq
     def __param_name_combinations(cls):
         required_param_names = sorted(cls.basic_allow_params())
-        for i in xrange(len(required_param_names)):
+        for i in range(len(required_param_names)):
             for some_param_names in itertools.combinations(required_param_names, i+1):
                 assert set(some_param_names).issubset(required_param_names)
                 yield list(some_param_names)
 
     @staticmethod
     def __adjust_params(params, kwargs):
         params.update(kwargs)
-        for name, value in list(params.iteritems()):
+        for name, value in list(params.items()):
             if value is None:
                 del params[name]
 

N6BrokerAuthApi/n6brokerauthapi/views.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2019 NASK. All rights reserved.
+# Copyright (c) 2013-2021 NASK. All rights reserved.
 
 import logging
 
@@ -21,7 +21,7 @@
 class _DenyAccess(Exception):
 
     def __init__(self, error_log_message=None):
-        super(_DenyAccess, self).__init__(error_log_message)
+        super().__init__(error_log_message)
         self.error_log_message = error_log_message
 
 
@@ -38,7 +38,7 @@ def __call__(self):
         try:
             # involves use of `iter_deduplicated_params()` and `make_response()`...
             try:
-                return super(_N6BrokerAuthViewBase, self).__call__()
+                return super().__call__()
             except ParamCleaningError as exc:
                 raise _DenyAccess(error_log_message=exc.public_message)
         except _DenyAccess as deny_exc:
@@ -81,7 +81,7 @@ def auth_manager_maker(self):
     @attr_required('param_name_to_required_flag')
     def get_required_param_names(cls):
         return {name
-                for name, required in cls.param_name_to_required_flag.iteritems()
+                for name, required in cls.param_name_to_required_flag.items()
                 if required}
 
     def allow_response(self):
@@ -100,18 +100,18 @@ def safe_name(self, name):
     # Private stuff
 
     def _log(self, level, log_message):
-        LOGGER.log(level, '[%r: %s] %s',
+        LOGGER.log(level, '[%a: %a] %a',
                    self,
                    ascii_str(self.request.url),
                    ascii_str(log_message))
 
     def _ensure_all_param_names_and_values_are_strings(self):
-        if not all(isinstance(key, basestring) and
-                   isinstance(val, basestring)
-                   for key, val in self.params.iteritems()):
+        if not all(isinstance(key, str) and
+                   isinstance(val, str)
+                   for key, val in self.params.items()):
             raise AssertionError(
                 'this should never happen: not all request param names and '
-                'values are strings! (params: {!r})'.format(self.params))
+                'values are strings! (params: {!a})'.format(self.params))
 
     def _warn_if_unknown_params(self):
         known_param_names = set(self.param_name_to_required_flag)
@@ -147,8 +147,8 @@ class _N6BrokerAuthResourceViewBase(_N6BrokerAuthViewBase):
 
     @attr_required('valid_permissions', 'valid_resources')
     def validate_params(self):
-        super(_N6BrokerAuthResourceViewBase, self).validate_params()
-        assert self.params.viewkeys() >= {'resource', 'permission'}
+        super().validate_params()
+        assert self.params.keys() >= {'resource', 'permission'}
         resource = self.params['resource']
         permission = self.params['permission']
         if resource not in self.valid_resources:

N6BrokerAuthApi/setup.py

@@ -21,16 +21,16 @@ def get_n6_version(filename_base):
         path = matching_paths[0]
     except IndexError:
         sys.exit('[{}] Cannot determine the n6 version '
-                 '(no files match the pattern {!r}).'
+                 '(no files match the pattern {!a}).'
                  .format(setup_human_readable_ref,
                          path_glob_pattern))
     try:
-        with open(path) as f:                                             #3: add: `, encoding='ascii'`
+        with open(path, encoding='ascii') as f:
             return f.read().strip()
     except (OSError, UnicodeError) as exc:
         sys.exit('[{}] Cannot determine the n6 version '
                  '(an error occurred when trying to '
-                 'read it from the file {!r} - {}).'
+                 'read it from the file {!a} - {}).'
                  .format(setup_human_readable_ref,
                          path,
                          exc))
@@ -41,7 +41,6 @@ def get_n6_version(filename_base):
 requires = [
     'n6lib==' + n6_version,
     'pyramid==1.10.8',
-    'typing',
 ]
 
 setup(
@@ -51,12 +50,13 @@ def get_n6_version(filename_base):
     packages=find_packages(),
     include_package_data=True,
     zip_safe=False,
+    python_requres='==3.9.*',
     install_requires=requires,
     entry_points="""\
         [paste.app_factory]
         main = n6brokerauthapi:main
     """,
-    tests_require=['mock==3.0.5', 'unittest_expander==0.3.1'],
+    tests_require=['unittest_expander==0.3.1'],
     test_suite='n6brokerauthapi.tests',
     description='Authentication and authorization API for RabbitMQ',
     url='https://github.com/CERT-Polska/n6',
@@ -66,7 +66,7 @@ def get_n6_version(filename_base):
         'License :: OSI Approved :: GNU Affero General Public License v3',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.9',
         "Framework :: Pyramid",
         'Topic :: Security',
     ],

N6Core/README.md

@@ -0,0 +1,8 @@
+**Note:** `N6Core` contains legacy *Python-2-only* stuff.  Typically,
+you will want to use -- instead of it -- the new, *Python-3-only* stuff
+residing in `N6DataPipeline`.
+
+Then it comes to data sources -- i.e., collectors and parsers --
+`N6DataSources` is the place where new sources should be implemented
+(in Python 3).  The collectors and parsers residing in `N6Core` will
+be gradually migrated to `N6DataSources` (if not obsolete).

N6Core/n6/archiver/recorder.py

@@ -9,6 +9,7 @@
 
 ### TODO: this module is to be replaced with a new implementation...
 
+from builtins import range  #3: --
 import datetime
 import logging
 import os
@@ -187,7 +188,7 @@ def _setup_db(self):
     def _install_session_variables_setter(self, engine, **session_variables):
         setter_sql = 'SET ' + ' , '.join(
             'SESSION {} = {}'.format(name, value)
-            for name, value in session_variables.iteritems())
+            for name, value in session_variables.items())
 
         @sqlalchemy.event.listens_for(engine, 'connect')
         def set_session_variables(dbapi_connection, connection_record):
@@ -270,7 +271,7 @@ def get_truncated_rk(rk, parts):
         rk = rk.split('.')
         parts_rk = []
         try:
-            for i in xrange(parts):
+            for i in range(parts):
                 parts_rk.append(rk[i])
         except IndexError:
             LOGGER.warning("routing key %r contains less than %r segments", rk, parts)

N6Core/n6/collectors/generic.py

@@ -35,6 +35,8 @@
     make_exc_ascii_str,
 )
 from n6corelib.email_message import ReceivedEmailMessage
+
+from n6lib.const import RAW_TYPE_ENUMS
 from n6lib.http_helpers import RequestPerformer
 from n6lib.log_helpers import (
     get_logger,
@@ -264,7 +266,6 @@ class BaseCollector(CollectorConfigMixin, QueuedBase, AbstractBaseCollector):
     # (note that this is something completely *different* than
     # <parser class>.event_type and <RecordDict instance>['type'])
     type = None
-    limits_type_of = ('stream', 'file', 'blacklist')
 
     # the attribute has to be overridden, if a component should
     # accept the "--n6recovery" argument option and inherits from
@@ -326,9 +327,9 @@ def set_queue_name(self):
 
     def _validate_type(self):
         """Validate type of message, should be one of: 'stream', 'file', 'blacklist."""
-        if self.type not in self.limits_type_of:
+        if self.type not in RAW_TYPE_ENUMS:
             raise Exception('Wrong type of archived data in mongo: {0},'
-                            '  should be one of: {1}'.format(self.type, self.limits_type_of))
+                            '  should be one of: {1}'.format(self.type, RAW_TYPE_ENUMS))
 
     def update_connection_params_dict_before_run(self, params_dict):
         """

N6Core/n6/data/conf/pipeline.conf → N6Core/n6/data/conf/00_pipeline.conf

@@ -25,3 +25,4 @@ comparator = enriched
 filter = enriched, compared
 anonymizer = filtered
 recorder = filtered
+counter= recorded

N6Core/n6/data/conf/07_aggregator.conf

@@ -1,15 +1,17 @@
 [aggregator]
 
-## path to the local aggregator's database file
-## (the database file will be created automatically
-## on the 1st aggregator run, if possible)
+# path to the local aggregator's database file
+# (the database file will be created automatically
+# on the 1st aggregator run, if possible)
 dbpath=~/.n6aggregator/aggregator_db.pickle
 
-## time interval (in seconds) within which non-monotonic times of
-## events are tolerated
+# time interval (in seconds) within which non-monotonic times of
+# events are tolerated
 time_tolerance=600
 
-## time interval like `time_tolerance`, but defined for specific source
-## (if it is not defined for the current source,
-## `time_tolerance` is used)
-time_tolerance_per_source={}
+# time interval like `time_tolerance`, but defined for specific source
+# (if it is not defined for the current source,
+# `time_tolerance` is used)
+;time_tolerance_per_source={
+;    "some-src.its-channel": 1200,
+;    "other-src.foobar": 900}