From f3a1a708455aeacdd78c2e33c66c9f5891f401eb Mon Sep 17 00:00:00 2001 From: CERT Polska Date: Fri, 22 Jun 2018 23:21:33 +0200 Subject: [PATCH] Initial release of n6 2.0 --- .gitignore | 30 + LICENSE.txt | 661 + N6AdminPanel/MANIFEST.in | 2 + N6AdminPanel/adminpanel.wsgi | 3 + N6AdminPanel/n6adminpanel/__init__.py | 0 N6AdminPanel/n6adminpanel/admin_panel.conf | 54 + N6AdminPanel/n6adminpanel/app.py | 602 + N6AdminPanel/n6adminpanel/static/logo.png | Bin 0 -> 11724 bytes N6AdminPanel/n6adminpanel/templates/home.html | 4 + N6AdminPanel/setup.py | 34 + N6Core/MANIFEST.in | 3 + N6Core/console_scripts | 6 + N6Core/console_scripts-nonpub | 8 + N6Core/n6/__init__.py | 3 + N6Core/n6/archiver/__init__.py | 0 N6Core/n6/archiver/archive_raw.py | 948 + N6Core/n6/archiver/mysqldb_patch.py | 171 + N6Core/n6/archiver/recorder.py | 441 + N6Core/n6/base/__init__.py | 0 N6Core/n6/base/config.py | 123 + N6Core/n6/base/queue.py | 1050 ++ N6Core/n6/collectors/__init__.py | 0 N6Core/n6/collectors/abuse_ch.py | 433 + N6Core/n6/collectors/badips.py | 59 + N6Core/n6/collectors/dns_bh.py | 35 + N6Core/n6/collectors/generic.py | 901 + N6Core/n6/collectors/greensnow.py | 35 + N6Core/n6/collectors/misp.py | 447 + N6Core/n6/collectors/packetmail.py | 55 + N6Core/n6/collectors/spam404.py | 31 + N6Core/n6/collectors/zone_h.py | 36 + N6Core/n6/data/conf/00_global.conf | 19 + N6Core/n6/data/conf/02_archiveraw.conf | 19 + N6Core/n6/data/conf/05_enrich.conf | 5 + N6Core/n6/data/conf/07_aggregator.conf | 10 + N6Core/n6/data/conf/07_comparator.conf | 9 + N6Core/n6/data/conf/09_auth_db.conf | 35 + N6Core/n6/data/conf/21_recorder.conf | 7 + N6Core/n6/data/conf/23_filter.conf | 2 + N6Core/n6/data/conf/70_abuse_ch.conf | 145 + N6Core/n6/data/conf/70_badips.conf | 11 + N6Core/n6/data/conf/70_dns_bh.conf | 12 + N6Core/n6/data/conf/70_greensnow.conf | 8 + N6Core/n6/data/conf/70_misp.conf | 29 + N6Core/n6/data/conf/70_packetmail.conf | 26 + N6Core/n6/data/conf/70_spam404.conf | 8 + N6Core/n6/data/conf/70_zone_h.conf | 10 + N6Core/n6/data/conf/logging.conf | 89 + N6Core/n6/parsers/__init__.py | 0 N6Core/n6/parsers/abuse_ch.py | 433 + N6Core/n6/parsers/badips.py | 31 + N6Core/n6/parsers/dns_bh.py | 61 + N6Core/n6/parsers/generic.py | 1039 ++ N6Core/n6/parsers/greensnow.py | 38 + N6Core/n6/parsers/misp.py | 170 + N6Core/n6/parsers/packetmail.py | 74 + N6Core/n6/parsers/spam404.py | 41 + N6Core/n6/parsers/zone_h.py | 115 + N6Core/n6/tests/__init__.py | 0 N6Core/n6/tests/archiver/__init__.py | 0 .../archiver/test_blacklist_compacter.py | 307 + N6Core/n6/tests/collectors/__init__.py | 0 N6Core/n6/tests/collectors/test_abuse_ch.py | 418 + N6Core/n6/tests/collectors/test_badips.py | 33 + N6Core/n6/tests/collectors/test_generic.py | 544 + N6Core/n6/tests/collectors/test_misp.py | 504 + N6Core/n6/tests/parsers/__init__.py | 0 N6Core/n6/tests/parsers/_parser_test_mixin.py | 207 + .../n6/tests/parsers/_parser_test_template.py | 229 + N6Core/n6/tests/parsers/test_abuse_ch.py | 684 + N6Core/n6/tests/parsers/test_badips.py | 57 + N6Core/n6/tests/parsers/test_dns_bh.py | 112 + N6Core/n6/tests/parsers/test_generic.py | 897 + N6Core/n6/tests/parsers/test_greensnow.py | 59 + N6Core/n6/tests/parsers/test_misp.py | 486 + N6Core/n6/tests/parsers/test_packetmail.py | 126 + N6Core/n6/tests/parsers/test_spam404.py | 59 + N6Core/n6/tests/parsers/test_zone_h.py | 99 + N6Core/n6/tests/utils/__init__.py | 0 N6Core/n6/tests/utils/test_aggregator.py | 1525 ++ N6Core/n6/tests/utils/test_comparator.py | 505 + N6Core/n6/tests/utils/test_enrich.py | 620 + N6Core/n6/tests/utils/test_filter.py | 1070 ++ N6Core/n6/utils/__init__.py | 0 N6Core/n6/utils/aggregator.py | 358 + N6Core/n6/utils/comparator.py | 436 + N6Core/n6/utils/enrich.py | 268 + N6Core/n6/utils/filter.py | 102 + N6Core/requirements | 1 + N6Core/setup.py | 127 + N6Lib/MANIFEST.in | 9 + N6Lib/n6lib/__init__.py | 27 + N6Lib/n6lib/_picklable_objs.py | 7 + N6Lib/n6lib/amqp_getters_pushers.py | 515 + N6Lib/n6lib/amqp_helpers.py | 100 + N6Lib/n6lib/argument_parser.py | 66 + N6Lib/n6lib/auth_api.py | 1270 ++ N6Lib/n6lib/auth_db/__init__.py | 0 N6Lib/n6lib/auth_db/config.py | 208 + N6Lib/n6lib/auth_db/fields.py | 42 + N6Lib/n6lib/auth_db/initialize_auth_db.py | 45 + N6Lib/n6lib/auth_db/models.py | 977 ++ N6Lib/n6lib/auth_db/populate_auth_db.py | 171 + N6Lib/n6lib/auth_db/validators.py | 164 + N6Lib/n6lib/auth_related_test_helpers.py | 1211 ++ N6Lib/n6lib/class_helpers.py | 173 + N6Lib/n6lib/common_helpers.py | 3196 ++++ N6Lib/n6lib/config.py | 4275 +++++ N6Lib/n6lib/const.py | 150 + N6Lib/n6lib/data_backend_api.py | 737 + N6Lib/n6lib/data_spec/__init__.py | 27 + N6Lib/n6lib/data_spec/_data_spec.py | 1141 ++ N6Lib/n6lib/data_spec/fields.py | 318 + N6Lib/n6lib/datetime_helpers.py | 53 + N6Lib/n6lib/db_events.py | 269 + N6Lib/n6lib/db_filtering_abstractions.py | 1492 ++ N6Lib/n6lib/email_message.py | 237 + N6Lib/n6lib/generate_test_events.py | 491 + N6Lib/n6lib/ldap_api_replacement.py | 1258 ++ N6Lib/n6lib/ldap_related_test_helpers.py | 339 + N6Lib/n6lib/log_helpers.py | 559 + N6Lib/n6lib/pyramid_commons/__init__.py | 54 + .../n6lib/pyramid_commons/_pyramid_commons.py | 677 + N6Lib/n6lib/pyramid_commons/renderers.py | 523 + N6Lib/n6lib/record_dict.py | 911 + .../n6lib/sqlalchemy_related_test_helpers.py | 20 + N6Lib/n6lib/tests/__init__.py | 0 N6Lib/n6lib/transaction_helpers.py | 118 + N6Lib/n6lib/unit_test_helpers.py | 433 + N6Lib/n6lib/unpacking_helpers.py | 67 + N6Lib/requirements | 27 + N6Lib/setup.py | 84 + N6Portal/MANIFEST.in | 2 + N6Portal/development.ini | 125 + N6Portal/gui/.babelrc | 15 + N6Portal/gui/.browserslistrc | 3 + N6Portal/gui/.editorconfig | 9 + N6Portal/gui/.eslintignore | 5 + N6Portal/gui/.eslintrc.js | 38 + N6Portal/gui/.gitignore | 17 + N6Portal/gui/.postcssrc.js | 9 + N6Portal/gui/README.md | 30 + N6Portal/gui/build/build.js | 41 + N6Portal/gui/build/check-versions.js | 49 + N6Portal/gui/build/logo.png | Bin 0 -> 6849 bytes N6Portal/gui/build/utils.js | 103 + N6Portal/gui/build/vue-loader.conf.js | 23 + N6Portal/gui/build/webpack.base.conf.js | 79 + N6Portal/gui/build/webpack.dev.conf.js | 78 + N6Portal/gui/build/webpack.prod.conf.js | 146 + N6Portal/gui/build/webpack.test.conf.js | 32 + N6Portal/gui/config/dev.env.js | 7 + N6Portal/gui/config/index.js | 81 + N6Portal/gui/config/prod.env.js | 4 + N6Portal/gui/config/test.env.js | 7 + N6Portal/gui/index.html | 14 + N6Portal/gui/package-lock.json | 14356 ++++++++++++++++ N6Portal/gui/package.json | 101 + N6Portal/gui/setup-install | 18 + N6Portal/gui/src/App.vue | 181 + .../components/AdminPanelNewCAAndLogin.vue | 36 + .../components/AdminPanelNewClientForm.vue | 379 + .../gui/src/components/AdminPanelPage.vue | 165 + N6Portal/gui/src/components/ErrorPage.vue | 23 + N6Portal/gui/src/components/LoginPage.vue | 175 + N6Portal/gui/src/components/MainPage.vue | 12 + .../gui/src/components/SearchCriterion.vue | 197 + N6Portal/gui/src/components/SearchPage.vue | 510 + N6Portal/gui/src/components/TheHeader.vue | 264 + N6Portal/gui/src/config/config.json | 4 + N6Portal/gui/src/images/logo.png | Bin 0 -> 11724 bytes N6Portal/gui/src/main.js | 43 + N6Portal/gui/src/router/index.js | 92 + N6Portal/gui/src/store/index.js | 77 + N6Portal/gui/static/.gitkeep | 0 .../e2e/custom-assertions/elementCount.js | 27 + N6Portal/gui/test/e2e/nightwatch.conf.js | 46 + N6Portal/gui/test/e2e/runner.js | 48 + N6Portal/gui/test/e2e/specs/test.js | 19 + N6Portal/gui/test/unit/.eslintrc | 9 + N6Portal/gui/test/unit/index.js | 13 + N6Portal/gui/test/unit/karma.conf.js | 33 + N6Portal/n6portal/__init__.py | 114 + N6Portal/production.ini | 161 + N6Portal/setup.py | 42 + N6RestApi/MANIFEST.in | 2 + N6RestApi/development.ini | 125 + N6RestApi/n6web/__init__.py | 225 + N6RestApi/n6web/tests/__init__.py | 0 N6RestApi/n6web/tests/test.py | 735 + N6RestApi/production.ini | 161 + N6RestApi/setup.py | 45 + N6SDK/ACKNOWLEDGEMENTS.txt | 7 + N6SDK/MANIFEST.in | 7 + N6SDK/NEWS.rst | 630 + N6SDK/docs/Makefile | 189 + N6SDK/docs/source/_static/.placeholder | 0 N6SDK/docs/source/_templates/.placeholder | 0 N6SDK/docs/source/api_test_tool.rst | 90 + N6SDK/docs/source/conf.py | 317 + N6SDK/docs/source/front_matter.rst | 1 + N6SDK/docs/source/index.rst | 40 + N6SDK/docs/source/lib_basic/data_spec.rst | 5 + .../source/lib_basic/data_spec_fields.rst | 5 + N6SDK/docs/source/lib_basic/exceptions.rst | 6 + .../docs/source/lib_basic/pyramid_commons.rst | 5 + .../lib_basic/pyramid_commons_renderers.rst | 5 + .../docs/source/lib_helpers/addr_helpers.rst | 4 + .../docs/source/lib_helpers/class_helpers.rst | 4 + .../source/lib_helpers/datetime_helpers.rst | 4 + .../source/lib_helpers/encoding_helpers.rst | 4 + N6SDK/docs/source/lib_helpers/regexes.rst | 4 + N6SDK/docs/source/library_reference.rst | 23 + N6SDK/docs/source/release_notes.rst | 5 + N6SDK/docs/source/tutorial.rst | 3107 ++++ N6SDK/n6sdk/__init__.py | 8 + N6SDK/n6sdk/_api_test_tool/__init__.py | 0 N6SDK/n6sdk/_api_test_tool/api_test_tool.py | 301 + N6SDK/n6sdk/_api_test_tool/client.py | 74 + N6SDK/n6sdk/_api_test_tool/config_base.ini | 27 + N6SDK/n6sdk/_api_test_tool/data_test.py | 40 + N6SDK/n6sdk/_api_test_tool/report.py | 64 + .../_api_test_tool/validator_exceptions.py | 6 + N6SDK/n6sdk/addr_helpers.py | 39 + N6SDK/n6sdk/class_helpers.py | 285 + N6SDK/n6sdk/data_spec/__init__.py | 67 + N6SDK/n6sdk/data_spec/_data_spec.py | 1050 ++ N6SDK/n6sdk/data_spec/fields.py | 1218 ++ N6SDK/n6sdk/datetime_helpers.py | 533 + N6SDK/n6sdk/encoding_helpers.py | 473 + N6SDK/n6sdk/exceptions.py | 471 + N6SDK/n6sdk/pyramid_commons/__init__.py | 53 + .../n6sdk/pyramid_commons/_pyramid_commons.py | 774 + N6SDK/n6sdk/pyramid_commons/renderers.py | 267 + N6SDK/n6sdk/regexes.py | 278 + N6SDK/n6sdk/scaffolds/__init__.py | 12 + .../+package+/__init__.py_tmpl | 103 + .../+package+/data_backend_api.py_tmpl | 116 + .../+package+/data_spec.py_tmpl | 161 + .../basic_n6sdk_scaffold/MANIFEST.in_tmpl | 2 + .../basic_n6sdk_scaffold/development.ini_tmpl | 60 + .../basic_n6sdk_scaffold/production.ini_tmpl | 54 + .../basic_n6sdk_scaffold/setup.py_tmpl | 38 + N6SDK/n6sdk/tests/__init__.py | 0 N6SDK/n6sdk/tests/_generic_helpers.py | 64 + N6SDK/n6sdk/tests/test_data_spec.py | 1117 ++ N6SDK/n6sdk/tests/test_data_spec_fields.py | 3884 +++++ N6SDK/n6sdk/tests/test_doctests.py | 34 + N6SDK/n6sdk/tests/test_pyramid_commons.py | 529 + N6SDK/n6sdk/tests/test_regexes.py | 430 + N6SDK/requirements | 5 + N6SDK/setup.py | 56 + README.rst | 52 + cef_logo.png | Bin 0 -> 21141 bytes do_setup.py | 190 + etc/apache2/.gitkeep | 0 etc/apache2/n6-adminpanel.conf | 37 + etc/apache2/n6-api.conf | 45 + etc/apache2/n6-portal-api.conf | 48 + etc/rabbitmq/rabbitmq.config | 29 + etc/sql/.gitkeep | 0 etc/sql/create_indexes.sql | 13 + etc/sql/create_tables.sql | 232 + etc/sql/mariadb.cnf | 166 + etc/ssl/generate_certs.sh | 21 + etc/ssl/openssl.cnf | 59 + etc/supervisord/get_parsers_conf.py | 52 + etc/supervisord/program_template.tmpl | 15 + etc/supervisord/programs/n6aggregator.conf | 15 + etc/supervisord/programs/n6archiveraw.conf | 15 + etc/supervisord/programs/n6comparator.conf | 15 + etc/supervisord/programs/n6enrich.conf | 16 + etc/supervisord/programs/n6filter.conf | 15 + etc/supervisord/programs/n6recorder.conf | 15 + etc/supervisord/supervisord.conf | 34 + test_do_setup.py | 455 + 276 files changed, 78944 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 N6AdminPanel/MANIFEST.in create mode 100644 N6AdminPanel/adminpanel.wsgi create mode 100644 N6AdminPanel/n6adminpanel/__init__.py create mode 100644 N6AdminPanel/n6adminpanel/admin_panel.conf create mode 100644 N6AdminPanel/n6adminpanel/app.py create mode 100644 N6AdminPanel/n6adminpanel/static/logo.png create mode 100644 N6AdminPanel/n6adminpanel/templates/home.html create mode 100644 N6AdminPanel/setup.py create mode 100644 N6Core/MANIFEST.in create mode 100644 N6Core/console_scripts create mode 100644 N6Core/console_scripts-nonpub create mode 100644 N6Core/n6/__init__.py create mode 100644 N6Core/n6/archiver/__init__.py create mode 100644 N6Core/n6/archiver/archive_raw.py create mode 100644 N6Core/n6/archiver/mysqldb_patch.py create mode 100644 N6Core/n6/archiver/recorder.py create mode 100644 N6Core/n6/base/__init__.py create mode 100644 N6Core/n6/base/config.py create mode 100644 N6Core/n6/base/queue.py create mode 100644 N6Core/n6/collectors/__init__.py create mode 100644 N6Core/n6/collectors/abuse_ch.py create mode 100644 N6Core/n6/collectors/badips.py create mode 100644 N6Core/n6/collectors/dns_bh.py create mode 100644 N6Core/n6/collectors/generic.py create mode 100644 N6Core/n6/collectors/greensnow.py create mode 100644 N6Core/n6/collectors/misp.py create mode 100644 N6Core/n6/collectors/packetmail.py create mode 100644 N6Core/n6/collectors/spam404.py create mode 100644 N6Core/n6/collectors/zone_h.py create mode 100644 N6Core/n6/data/conf/00_global.conf create mode 100644 N6Core/n6/data/conf/02_archiveraw.conf create mode 100644 N6Core/n6/data/conf/05_enrich.conf create mode 100644 N6Core/n6/data/conf/07_aggregator.conf create mode 100644 N6Core/n6/data/conf/07_comparator.conf create mode 100644 N6Core/n6/data/conf/09_auth_db.conf create mode 100644 N6Core/n6/data/conf/21_recorder.conf create mode 100644 N6Core/n6/data/conf/23_filter.conf create mode 100644 N6Core/n6/data/conf/70_abuse_ch.conf create mode 100644 N6Core/n6/data/conf/70_badips.conf create mode 100644 N6Core/n6/data/conf/70_dns_bh.conf create mode 100644 N6Core/n6/data/conf/70_greensnow.conf create mode 100644 N6Core/n6/data/conf/70_misp.conf create mode 100644 N6Core/n6/data/conf/70_packetmail.conf create mode 100644 N6Core/n6/data/conf/70_spam404.conf create mode 100644 N6Core/n6/data/conf/70_zone_h.conf create mode 100644 N6Core/n6/data/conf/logging.conf create mode 100644 N6Core/n6/parsers/__init__.py create mode 100644 N6Core/n6/parsers/abuse_ch.py create mode 100644 N6Core/n6/parsers/badips.py create mode 100644 N6Core/n6/parsers/dns_bh.py create mode 100644 N6Core/n6/parsers/generic.py create mode 100644 N6Core/n6/parsers/greensnow.py create mode 100644 N6Core/n6/parsers/misp.py create mode 100644 N6Core/n6/parsers/packetmail.py create mode 100644 N6Core/n6/parsers/spam404.py create mode 100644 N6Core/n6/parsers/zone_h.py create mode 100644 N6Core/n6/tests/__init__.py create mode 100644 N6Core/n6/tests/archiver/__init__.py create mode 100644 N6Core/n6/tests/archiver/test_blacklist_compacter.py create mode 100644 N6Core/n6/tests/collectors/__init__.py create mode 100644 N6Core/n6/tests/collectors/test_abuse_ch.py create mode 100644 N6Core/n6/tests/collectors/test_badips.py create mode 100644 N6Core/n6/tests/collectors/test_generic.py create mode 100644 N6Core/n6/tests/collectors/test_misp.py create mode 100644 N6Core/n6/tests/parsers/__init__.py create mode 100644 N6Core/n6/tests/parsers/_parser_test_mixin.py create mode 100644 N6Core/n6/tests/parsers/_parser_test_template.py create mode 100644 N6Core/n6/tests/parsers/test_abuse_ch.py create mode 100644 N6Core/n6/tests/parsers/test_badips.py create mode 100644 N6Core/n6/tests/parsers/test_dns_bh.py create mode 100644 N6Core/n6/tests/parsers/test_generic.py create mode 100644 N6Core/n6/tests/parsers/test_greensnow.py create mode 100644 N6Core/n6/tests/parsers/test_misp.py create mode 100644 N6Core/n6/tests/parsers/test_packetmail.py create mode 100644 N6Core/n6/tests/parsers/test_spam404.py create mode 100644 N6Core/n6/tests/parsers/test_zone_h.py create mode 100644 N6Core/n6/tests/utils/__init__.py create mode 100644 N6Core/n6/tests/utils/test_aggregator.py create mode 100644 N6Core/n6/tests/utils/test_comparator.py create mode 100644 N6Core/n6/tests/utils/test_enrich.py create mode 100644 N6Core/n6/tests/utils/test_filter.py create mode 100644 N6Core/n6/utils/__init__.py create mode 100644 N6Core/n6/utils/aggregator.py create mode 100644 N6Core/n6/utils/comparator.py create mode 100644 N6Core/n6/utils/enrich.py create mode 100644 N6Core/n6/utils/filter.py create mode 100644 N6Core/requirements create mode 100644 N6Core/setup.py create mode 100644 N6Lib/MANIFEST.in create mode 100644 N6Lib/n6lib/__init__.py create mode 100644 N6Lib/n6lib/_picklable_objs.py create mode 100644 N6Lib/n6lib/amqp_getters_pushers.py create mode 100644 N6Lib/n6lib/amqp_helpers.py create mode 100644 N6Lib/n6lib/argument_parser.py create mode 100644 N6Lib/n6lib/auth_api.py create mode 100644 N6Lib/n6lib/auth_db/__init__.py create mode 100644 N6Lib/n6lib/auth_db/config.py create mode 100644 N6Lib/n6lib/auth_db/fields.py create mode 100644 N6Lib/n6lib/auth_db/initialize_auth_db.py create mode 100644 N6Lib/n6lib/auth_db/models.py create mode 100644 N6Lib/n6lib/auth_db/populate_auth_db.py create mode 100644 N6Lib/n6lib/auth_db/validators.py create mode 100644 N6Lib/n6lib/auth_related_test_helpers.py create mode 100644 N6Lib/n6lib/class_helpers.py create mode 100644 N6Lib/n6lib/common_helpers.py create mode 100644 N6Lib/n6lib/config.py create mode 100644 N6Lib/n6lib/const.py create mode 100644 N6Lib/n6lib/data_backend_api.py create mode 100644 N6Lib/n6lib/data_spec/__init__.py create mode 100644 N6Lib/n6lib/data_spec/_data_spec.py create mode 100644 N6Lib/n6lib/data_spec/fields.py create mode 100644 N6Lib/n6lib/datetime_helpers.py create mode 100644 N6Lib/n6lib/db_events.py create mode 100644 N6Lib/n6lib/db_filtering_abstractions.py create mode 100644 N6Lib/n6lib/email_message.py create mode 100644 N6Lib/n6lib/generate_test_events.py create mode 100644 N6Lib/n6lib/ldap_api_replacement.py create mode 100644 N6Lib/n6lib/ldap_related_test_helpers.py create mode 100644 N6Lib/n6lib/log_helpers.py create mode 100644 N6Lib/n6lib/pyramid_commons/__init__.py create mode 100644 N6Lib/n6lib/pyramid_commons/_pyramid_commons.py create mode 100644 N6Lib/n6lib/pyramid_commons/renderers.py create mode 100644 N6Lib/n6lib/record_dict.py create mode 100644 N6Lib/n6lib/sqlalchemy_related_test_helpers.py create mode 100644 N6Lib/n6lib/tests/__init__.py create mode 100644 N6Lib/n6lib/transaction_helpers.py create mode 100644 N6Lib/n6lib/unit_test_helpers.py create mode 100644 N6Lib/n6lib/unpacking_helpers.py create mode 100644 N6Lib/requirements create mode 100644 N6Lib/setup.py create mode 100644 N6Portal/MANIFEST.in create mode 100644 N6Portal/development.ini create mode 100644 N6Portal/gui/.babelrc create mode 100644 N6Portal/gui/.browserslistrc create mode 100644 N6Portal/gui/.editorconfig create mode 100644 N6Portal/gui/.eslintignore create mode 100644 N6Portal/gui/.eslintrc.js create mode 100644 N6Portal/gui/.gitignore create mode 100644 N6Portal/gui/.postcssrc.js create mode 100644 N6Portal/gui/README.md create mode 100644 N6Portal/gui/build/build.js create mode 100644 N6Portal/gui/build/check-versions.js create mode 100644 N6Portal/gui/build/logo.png create mode 100644 N6Portal/gui/build/utils.js create mode 100644 N6Portal/gui/build/vue-loader.conf.js create mode 100644 N6Portal/gui/build/webpack.base.conf.js create mode 100644 N6Portal/gui/build/webpack.dev.conf.js create mode 100644 N6Portal/gui/build/webpack.prod.conf.js create mode 100644 N6Portal/gui/build/webpack.test.conf.js create mode 100644 N6Portal/gui/config/dev.env.js create mode 100644 N6Portal/gui/config/index.js create mode 100644 N6Portal/gui/config/prod.env.js create mode 100644 N6Portal/gui/config/test.env.js create mode 100644 N6Portal/gui/index.html create mode 100644 N6Portal/gui/package-lock.json create mode 100644 N6Portal/gui/package.json create mode 100644 N6Portal/gui/setup-install create mode 100644 N6Portal/gui/src/App.vue create mode 100644 N6Portal/gui/src/components/AdminPanelNewCAAndLogin.vue create mode 100644 N6Portal/gui/src/components/AdminPanelNewClientForm.vue create mode 100644 N6Portal/gui/src/components/AdminPanelPage.vue create mode 100644 N6Portal/gui/src/components/ErrorPage.vue create mode 100644 N6Portal/gui/src/components/LoginPage.vue create mode 100644 N6Portal/gui/src/components/MainPage.vue create mode 100644 N6Portal/gui/src/components/SearchCriterion.vue create mode 100644 N6Portal/gui/src/components/SearchPage.vue create mode 100644 N6Portal/gui/src/components/TheHeader.vue create mode 100644 N6Portal/gui/src/config/config.json create mode 100644 N6Portal/gui/src/images/logo.png create mode 100644 N6Portal/gui/src/main.js create mode 100644 N6Portal/gui/src/router/index.js create mode 100644 N6Portal/gui/src/store/index.js create mode 100644 N6Portal/gui/static/.gitkeep create mode 100644 N6Portal/gui/test/e2e/custom-assertions/elementCount.js create mode 100644 N6Portal/gui/test/e2e/nightwatch.conf.js create mode 100644 N6Portal/gui/test/e2e/runner.js create mode 100644 N6Portal/gui/test/e2e/specs/test.js create mode 100644 N6Portal/gui/test/unit/.eslintrc create mode 100644 N6Portal/gui/test/unit/index.js create mode 100644 N6Portal/gui/test/unit/karma.conf.js create mode 100644 N6Portal/n6portal/__init__.py create mode 100644 N6Portal/production.ini create mode 100644 N6Portal/setup.py create mode 100644 N6RestApi/MANIFEST.in create mode 100644 N6RestApi/development.ini create mode 100644 N6RestApi/n6web/__init__.py create mode 100644 N6RestApi/n6web/tests/__init__.py create mode 100644 N6RestApi/n6web/tests/test.py create mode 100644 N6RestApi/production.ini create mode 100644 N6RestApi/setup.py create mode 100644 N6SDK/ACKNOWLEDGEMENTS.txt create mode 100644 N6SDK/MANIFEST.in create mode 100644 N6SDK/NEWS.rst create mode 100644 N6SDK/docs/Makefile create mode 100644 N6SDK/docs/source/_static/.placeholder create mode 100644 N6SDK/docs/source/_templates/.placeholder create mode 100644 N6SDK/docs/source/api_test_tool.rst create mode 100644 N6SDK/docs/source/conf.py create mode 100644 N6SDK/docs/source/front_matter.rst create mode 100644 N6SDK/docs/source/index.rst create mode 100644 N6SDK/docs/source/lib_basic/data_spec.rst create mode 100644 N6SDK/docs/source/lib_basic/data_spec_fields.rst create mode 100644 N6SDK/docs/source/lib_basic/exceptions.rst create mode 100644 N6SDK/docs/source/lib_basic/pyramid_commons.rst create mode 100644 N6SDK/docs/source/lib_basic/pyramid_commons_renderers.rst create mode 100644 N6SDK/docs/source/lib_helpers/addr_helpers.rst create mode 100644 N6SDK/docs/source/lib_helpers/class_helpers.rst create mode 100644 N6SDK/docs/source/lib_helpers/datetime_helpers.rst create mode 100644 N6SDK/docs/source/lib_helpers/encoding_helpers.rst create mode 100644 N6SDK/docs/source/lib_helpers/regexes.rst create mode 100644 N6SDK/docs/source/library_reference.rst create mode 100644 N6SDK/docs/source/release_notes.rst create mode 100644 N6SDK/docs/source/tutorial.rst create mode 100644 N6SDK/n6sdk/__init__.py create mode 100644 N6SDK/n6sdk/_api_test_tool/__init__.py create mode 100644 N6SDK/n6sdk/_api_test_tool/api_test_tool.py create mode 100644 N6SDK/n6sdk/_api_test_tool/client.py create mode 100644 N6SDK/n6sdk/_api_test_tool/config_base.ini create mode 100644 N6SDK/n6sdk/_api_test_tool/data_test.py create mode 100644 N6SDK/n6sdk/_api_test_tool/report.py create mode 100644 N6SDK/n6sdk/_api_test_tool/validator_exceptions.py create mode 100644 N6SDK/n6sdk/addr_helpers.py create mode 100644 N6SDK/n6sdk/class_helpers.py create mode 100644 N6SDK/n6sdk/data_spec/__init__.py create mode 100644 N6SDK/n6sdk/data_spec/_data_spec.py create mode 100644 N6SDK/n6sdk/data_spec/fields.py create mode 100644 N6SDK/n6sdk/datetime_helpers.py create mode 100644 N6SDK/n6sdk/encoding_helpers.py create mode 100644 N6SDK/n6sdk/exceptions.py create mode 100644 N6SDK/n6sdk/pyramid_commons/__init__.py create mode 100644 N6SDK/n6sdk/pyramid_commons/_pyramid_commons.py create mode 100644 N6SDK/n6sdk/pyramid_commons/renderers.py create mode 100644 N6SDK/n6sdk/regexes.py create mode 100644 N6SDK/n6sdk/scaffolds/__init__.py create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/+package+/__init__.py_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/+package+/data_backend_api.py_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/+package+/data_spec.py_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/MANIFEST.in_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/development.ini_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/production.ini_tmpl create mode 100644 N6SDK/n6sdk/scaffolds/basic_n6sdk_scaffold/setup.py_tmpl create mode 100644 N6SDK/n6sdk/tests/__init__.py create mode 100644 N6SDK/n6sdk/tests/_generic_helpers.py create mode 100644 N6SDK/n6sdk/tests/test_data_spec.py create mode 100644 N6SDK/n6sdk/tests/test_data_spec_fields.py create mode 100644 N6SDK/n6sdk/tests/test_doctests.py create mode 100644 N6SDK/n6sdk/tests/test_pyramid_commons.py create mode 100644 N6SDK/n6sdk/tests/test_regexes.py create mode 100644 N6SDK/requirements create mode 100644 N6SDK/setup.py create mode 100644 README.rst create mode 100644 cef_logo.png create mode 100755 do_setup.py create mode 100644 etc/apache2/.gitkeep create mode 100644 etc/apache2/n6-adminpanel.conf create mode 100644 etc/apache2/n6-api.conf create mode 100644 etc/apache2/n6-portal-api.conf create mode 100644 etc/rabbitmq/rabbitmq.config create mode 100644 etc/sql/.gitkeep create mode 100644 etc/sql/create_indexes.sql create mode 100644 etc/sql/create_tables.sql create mode 100644 etc/sql/mariadb.cnf create mode 100755 etc/ssl/generate_certs.sh create mode 100644 etc/ssl/openssl.cnf create mode 100644 etc/supervisord/get_parsers_conf.py create mode 100644 etc/supervisord/program_template.tmpl create mode 100644 etc/supervisord/programs/n6aggregator.conf create mode 100644 etc/supervisord/programs/n6archiveraw.conf create mode 100644 etc/supervisord/programs/n6comparator.conf create mode 100644 etc/supervisord/programs/n6enrich.conf create mode 100644 etc/supervisord/programs/n6filter.conf create mode 100644 etc/supervisord/programs/n6recorder.conf create mode 100644 etc/supervisord/supervisord.conf create mode 100755 test_do_setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..959703b --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +*.pyc +*.py~ +*.js~ +*.egg +*.sqlite +*_flymake.py + +.coverage +nosetests.xml + +.project +.idea +.pydevproject +.ropeproject +.settings +.dir-locals.el +.emacs.desktop +.emacs.desktop.lock + +[Nn]6*.egg-info/ + +/N6Portal/*.ini~ +/N6Portal/*_local.ini +/N6RestApi/*.ini~ +/N6RestApi/*_local.ini + +/N6*/build/ +/N6*/dist/ +/N6*/docs/build/ + diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/N6AdminPanel/MANIFEST.in b/N6AdminPanel/MANIFEST.in new file mode 100644 index 0000000..d4ecfd4 --- /dev/null +++ b/N6AdminPanel/MANIFEST.in @@ -0,0 +1,2 @@ +include *.txt *.rst *.conf *.wsgi +recursive-include n6adminpanel *.conf *.html *.png diff --git a/N6AdminPanel/adminpanel.wsgi b/N6AdminPanel/adminpanel.wsgi new file mode 100644 index 0000000..b851c8c --- /dev/null +++ b/N6AdminPanel/adminpanel.wsgi @@ -0,0 +1,3 @@ +#!/usr/bin/python +from n6adminpanel.app import get_app +application = get_app() diff --git a/N6AdminPanel/n6adminpanel/__init__.py b/N6AdminPanel/n6adminpanel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/N6AdminPanel/n6adminpanel/admin_panel.conf b/N6AdminPanel/n6adminpanel/admin_panel.conf new file mode 100644 index 0000000..f37344f --- /dev/null +++ b/N6AdminPanel/n6adminpanel/admin_panel.conf @@ -0,0 +1,54 @@ +# IMPORTANT: this file should be copied to the "~/.n6" or "/etc/n6" +# directory and adjusted appropriately -- SEE THE COMMENTS BELOW... + + + +[admin_panel] + +## the value of `app_secret_key` must be set to some unpredictable +## secret -- you can generate it with the command: +## python -c 'import os, base64; print(base64.b64encode(os.urandom(16)))' +#app_secret_key = + + + +## IMPORTANT: the following 3 config sections should be uncommented +## and adjusted *ONLY* if the n6 Admin Panel application does *NOT* +## have access to the 09_auth_db.conf (being part of the N6Core +## configuration) which already contains these sections! +# +#[auth_db] +# +## connection URL, e.g.: mysql+mysqldb://n6:somepassword@localhost/n6 +## it must start with `mysql+mysqldb:` (or just `mysql:`) because other +## dialects/drivers are not supported +#url = mysql://user:password@host/dbname +# +## if you want to use SSL, the following options must be set to +## appropriate file paths: +#ssl_cacert = /some/path/to/CACertificatesFile.pem +#ssl_cert = /some/path/to/ClientCertificateFile.pem +#ssl_key = /some/path/to/private/ClientCertificateKeyFile.pem +# +# +#[auth_db_session_variables] +# +## all MySQL variables specified within this section will be set by +## executing "SET SESSION = , ...". +## WARNING: for simplicity, the variable names and values are inserted +## into SQL code "as is", *without* any escaping (we assume we can treat +## configuration files as a *trusted* source of data). +# +## (`[auth_db_session_variables].wait_timeout` should be +## greater than `[auth_db_connection_pool].pool_recycle`) +#wait_timeout = 7200 +# +# +#[auth_db_connection_pool] +# +## (generally, the defaults should be OK in most cases; if you are +## interested in technical details -- see: SQLAlchemy docs...) +#pool_recycle = 3600 +#pool_timeout = 20 +#pool_size = 15 +#max_overflow = 12 diff --git a/N6AdminPanel/n6adminpanel/app.py b/N6AdminPanel/n6adminpanel/app.py new file mode 100644 index 0000000..bd0efd8 --- /dev/null +++ b/N6AdminPanel/n6adminpanel/app.py @@ -0,0 +1,602 @@ +# Copyright (c) 2013-2018 NASK. All rights reserved. + +from collections import MutableSequence + +from flask import Flask +from flask_admin import ( + Admin, + AdminIndexView, + expose, +) +from flask_admin._compat import iteritems +from flask_admin.contrib.sqla import ( + form, + ModelView, +) +from flask_admin.contrib.sqla.form import InlineModelConverter +from flask_admin.form import ( + TimeField, + rules, +) +from flask_admin.form.widgets import TimePickerWidget +from flask_admin.model.form import ( + InlineFormAdmin, + converts, +) +from flask_admin.model.fields import InlineModelFormField +from sqlalchemy import inspect +from sqlalchemy.sql.sqltypes import String +from wtforms import PasswordField +from wtforms.fields import Field + +from n6lib.config import ConfigMixin +from n6lib.common_helpers import as_unicode +from n6lib.log_helpers import logging_configured +from n6lib.auth_db.models import ( + CACert, + Cert, + #Component, + CriteriaASN, + CriteriaCC, + #CriteriaCategory, + CriteriaContainer, + CriteriaIPNetwork, + CriteriaName, + EMailNotificationAddress, + EMailNotificationTime, + InsideFilterASN, + InsideFilterCC, + InsideFilterFQDN, + InsideFilterIPNetwork, + InsideFilterURL, + Org, + OrgGroup, + RequestCase, + Source, + Subsource, + SubsourceGroup, + SystemGroup, + User, + db_session, +) +from n6lib.auth_db.config import SQLAuthDBConfigMixin +from n6sdk.exceptions import FieldValueError + + +class _PasswordFieldHandlerMixin(object): + + form_extra_fields = { + 'password': PasswordField(), + } + + def on_model_change(self, form, model, is_created): + if form.password and form.password.data: + model.password = model.get_password_hash_or_none(form.password.data) + + +class PrimaryKeyOnlyFormAdmin(InlineFormAdmin): + + column_display_pk = True + + def _get_form_columns(self, model): + inspection = inspect(model) + return [inspection.primary_key[0].name] + + def __init__(self, model, **kwargs): + self.form_columns = self._get_form_columns(model) + super(PrimaryKeyOnlyFormAdmin, self).__init__(model, **kwargs) + + +class InlineMappingFormAdmin(PrimaryKeyOnlyFormAdmin): + + """ + Extended Flask-admin's `InlineFormAdmin` class, that allows + to define a custom mapping of inline forms inside a view. + + Original class creates only one form from each model listed + in the `inline_models` attribute. This class overrides the + behavior, allowing to create more than one inline form + from a single model. + + Its constructor accepts additional argument - `inline_mapping`, + which has to be a dict mapping names of model's relationship + fields to corresponding field names in related models. + + Let us take some example relations (m:n for simplicity): + ModelOne.rel_with_model_two - ModelTwo.rel_with_model_one + ModelOne.another_rel_with_model_two - ModelTwo.another_rel_with_model_one + ModelOne.rel_with_model_three - ModelThree.rel_with_model_one + Then the `inline_models` attribute of `ModelOne` should be + created like this: + + inline_models = [ + InlineMappingFormAdmin({ + 'rel_with_model_two': 'rel_with_model_one', + 'another_rel_with_model_two': 'another_rel_with_model_one', + }, ModelTwo), + InlineMappingFormAdmin({ + 'rel_with_model_three': 'rel_with_model_one', + }, ModelThree), + ] + """ + + column_display_pk = True + + def __init__(self, inline_mapping, model, **kwargs): + self.inline_mapping = inline_mapping + super(InlineMappingFormAdmin, self).__init__(model, **kwargs) + + +class UserInlineFormAdmin(_PasswordFieldHandlerMixin, InlineFormAdmin): + + column_display_pk = True + column_descriptions = { + 'login': 'User\'s login (e-mail address)', + } + form_columns = ['login', 'password'] + + +class NotificationTimeInlineFormAdmin(InlineFormAdmin): + + form_args = { + 'notification_time': { + 'default_format': '%H:%M', + }, + } + + +class SubsourceInlineFormAdmin(InlineFormAdmin): + + column_display_pk = True + form_columns = [ + 'label', + 'inclusion_criteria', + 'exclusion_criteria', + 'subsource_groups', + 'inside_org_groups', + 'threats_org_groups', + 'search_org_groups', + 'inside_orgs', + 'inside_ex_orgs', + 'threats_orgs', + 'threats_ex_orgs', + 'search_orgs', + 'search_ex_orgs', + ] + + +class CustomColumnListView(ModelView): + + def _set_list_of_form_columns(self, model): + pk_columns = [] + fk_columns = [] + sorted_columns = [] + inspection = inspect(model) + for pk in inspection.primary_key: + pk_columns.append(pk.name) + fk_constraints = model.__table__.foreign_keys + for fk in fk_constraints: + if hasattr(fk, 'constraint') and hasattr(fk.constraint, 'columns'): + fk_columns.extend([column.name for column in fk.constraint.columns]) + all_columns = inspection.columns.keys() + regular_columns = list(set(all_columns) - set(fk_columns) - set(pk_columns)) + sorted_columns.extend(pk_columns) + sorted_columns.extend(regular_columns) + self.form_columns = sorted_columns + relationships = inspection.relationships.keys() + self.form_columns.extend(relationships) + + column_display_pk = True + + def __init__(self, model, session, **kwargs): + self._set_list_of_form_columns(model) + super(CustomColumnListView, self).__init__(model, session, **kwargs) + + +class PatchedInlineModelFormField(InlineModelFormField): + + def populate_obj(self, obj, name): + string_pk_models = getattr(self, 'string_pk_models', None) + check_for_pk = True + if string_pk_models and any([x for x in string_pk_models if isinstance(obj, x)]): + check_for_pk = False + for name, field in iteritems(self.form._fields): + if not check_for_pk or name != self._pk: + field.populate_obj(obj, name) + + +class PatchedInlineFieldListType(form.InlineModelFormList): + + form_field_type = PatchedInlineModelFormField + + +class PatchedInlineModelConverter(InlineModelConverter): + + inline_field_list_type = PatchedInlineFieldListType + + def __init__(self, *args): + self._calculated_key_pair = None + self._original_calculate_mapping_meth = self._calculate_mapping_key_pair + self._calculate_mapping_key_pair = self._new_calculate_mapping_meth + super(PatchedInlineModelConverter, self).__init__(*args) + + def _new_calculate_mapping_meth(self, *args): + return self._calculated_key_pair + + def _patched_calculate_mapping_meth(self, model, info): + if hasattr(info, 'inline_mapping'): + for forward, reverse in info.inline_mapping.iteritems(): + yield forward, reverse + else: + yield self._original_calculate_mapping_meth(model, info) + + def contribute(self, model, form_class, inline_model): + info = self.get_info(inline_model) + contribute_result = None + for calculated_pair in self._patched_calculate_mapping_meth(model, info): + self._calculated_key_pair = calculated_pair + contribute_result = super(PatchedInlineModelConverter, self).contribute(model, + form_class, + inline_model) + return contribute_result + + +class ShowInlineStringPKModelView(ModelView): + + inline_model_form_converter = PatchedInlineModelConverter + + def __init__(self, model, session, inline_string_pk_models=None, **kwargs): + if inline_string_pk_models: + self.inline_string_pk_models = inline_string_pk_models + super(ShowInlineStringPKModelView, self).__init__(model, session, **kwargs) + + def create_model(self, form): + if self.inline_string_pk_models: + setattr(self.inline_model_form_converter.inline_field_list_type.form_field_type, + 'string_pk_models', + self.inline_string_pk_models) + return super(ShowInlineStringPKModelView, self).create_model(form) + + +class ShortTimePickerWidget(TimePickerWidget): + + """ + Widget class extended in order to adjust time format, saved + into input field by the time picking widget. There is no need + to save seconds. + """ + + def __call__(self, field, **kwargs): + kwargs['data-date-format'] = u'HH:mm' + return super(ShortTimePickerWidget, self).__call__(field, **kwargs) + + +class ShortTimeField(TimeField): + + widget = ShortTimePickerWidget() + + +class OrgModelConverter(form.AdminModelConverter): + + @converts('Time') + def convert_time(self, field_args, **extra): + return ShortTimeField(**field_args) + + +class OrgView(ShowInlineStringPKModelView): + + # create_modal = True + # edit_modal = True + model_form_converter = OrgModelConverter + column_descriptions = { + 'org_id': 'Organization identifier', + } + can_view_details = True + # essential to display PK column in the "list" view + column_display_pk = True + column_searchable_list = ['org_id'] + column_list = [ + 'org_id', + 'full_access', + # 'stream_api_enabled', + # 'email_notifications_enabled', + # 'email_notifications_business_days_only', + 'access_to_inside', + 'access_to_threats', + 'access_to_search', + ] + form_columns = [ + 'org_id', + 'org_groups', + 'users', + 'full_access', + 'access_to_inside', + # 'inside_max_days_old', + # 'inside_request_parameters', + 'inside_subsources', + 'inside_ex_subsources', + 'inside_subsource_groups', + 'inside_ex_subsource_groups', + 'access_to_threats', + # 'threats_max_days_old', + # 'threats_request_parameters', + 'threats_subsources', + 'threats_ex_subsources', + 'threats_subsource_groups', + 'threats_ex_subsource_groups', + 'access_to_search', + # 'search_max_days_old', + # 'search_request_parameters', + 'search_subsources', + 'search_ex_subsources', + 'search_subsource_groups', + 'search_ex_subsource_groups', + # other options/notifications settings + # 'stream_api_enabled', + # 'email_notifications_enabled', + # 'email_notifications_addresses', + # 'email_notifications_times', + # 'email_notifications_language', + # 'email_notifications_business_days_only', + 'inside_filter_asns', + 'inside_filter_ccs', + 'inside_filter_fqdns', + 'inside_filter_ip_networks', + 'inside_filter_urls', + ] + form_rules = [ + rules.Header('Basic options for organization'), + rules.Field('org_id'), + rules.Field('org_groups'), + rules.Field('full_access'), + rules.Header('Users'), + rules.Field('users'), + rules.Header('"Inside" resource'), + rules.Field('access_to_inside'), + # rules.Field('inside_max_days_old'), + # rules.Field('inside_request_parameters'), + rules.Field('inside_subsources'), + rules.Field('inside_ex_subsources'), + rules.Field('inside_subsource_groups'), + rules.Field('inside_ex_subsource_groups'), + rules.Header('"Threats" resource'), + rules.Field('access_to_threats'), + # rules.Field('threats_max_days_old'), + # rules.Field('threats_request_parameters'), + rules.Field('threats_subsources'), + rules.Field('threats_ex_subsources'), + rules.Field('threats_subsource_groups'), + rules.Field('threats_ex_subsource_groups'), + rules.Header('"Search" resource'), + rules.Field('access_to_search'), + # rules.Field('search_max_days_old'), + # rules.Field('search_request_parameters'), + rules.Field('search_subsources'), + rules.Field('search_ex_subsources'), + rules.Field('search_subsource_groups'), + rules.Field('search_ex_subsource_groups'), + # rules.Header('Other options'), + # rules.Field('stream_api_enabled'), + # rules.Field('email_notifications_enabled'), + # rules.Field('email_notifications_addresses'), + # rules.Field('email_notifications_times'), + # rules.Field('email_notifications_language'), + # rules.Field('email_notifications_business_days_only'), + rules.Header('Criteria for "Inside" (n6filter)'), + rules.Field('inside_filter_asns'), + rules.Field('inside_filter_ccs'), + rules.Field('inside_filter_fqdns'), + rules.Field('inside_filter_ip_networks'), + rules.Field('inside_filter_urls'), + ] + inline_models = [ + UserInlineFormAdmin(User), + EMailNotificationAddress, + NotificationTimeInlineFormAdmin(EMailNotificationTime), + InsideFilterASN, + InsideFilterCC, + InsideFilterFQDN, + InsideFilterIPNetwork, + InsideFilterURL, + ] + + +class UserView(_PasswordFieldHandlerMixin, ModelView): + + column_descriptions = { + 'login': 'User\'s login (e-mail address)', + } + column_list = ['login', 'org', 'system_groups'] + form_columns = ['login', 'password', 'org', 'system_groups'] + # column list including certificate-related columns + # column_list = ['login', 'org', 'system_groups', 'created_certs', 'owned_certs', + # 'revoked_certs', 'sent_request_cases'] + + +class ComponentView(_PasswordFieldHandlerMixin, ModelView): + + column_list = ['login'] + form_columns = ['login', 'password'] + # column list including certificate-related columns + # column_list = ['login', 'created_certs', 'owned_certs', 'revoked_certs'] + + +class CriteriaContainerView(CustomColumnListView): + + inline_models = [ + CriteriaASN, + CriteriaCC, + CriteriaIPNetwork, + CriteriaName, + ] + + +class SourceView(ShowInlineStringPKModelView, CustomColumnListView): + + inline_models = [ + SubsourceInlineFormAdmin(Subsource), + ] + + +class CustomIndexView(AdminIndexView): + + @expose('/') + def index(self): + return self.render('home.html') + + +class AdminPanel(ConfigMixin): + + config_spec = ''' + [admin_panel] + app_secret_key + app_name = n6 Admin Panel + template_mode = bootstrap3 + ''' + engine_config_prefix = '' + string_pk_table_models_views = [ + (Org, OrgView, {'inline_string_pk_models': [User]}), + (OrgGroup, CustomColumnListView, None), + (User, UserView, None), + #(Component, ComponentView, None), + (CriteriaContainer, CriteriaContainerView, None), + #(CriteriaCategory, CustomColumnListView, None), + (Source, SourceView, {'inline_string_pk_models': [Subsource]}), + (Subsource, CustomColumnListView, None), + (SubsourceGroup, CustomColumnListView, None), + (SystemGroup, CustomColumnListView, None), + ] + # list of models with single, main column, which primary keys + # are auto-generated integers + auto_pk_table_classes = [ + CriteriaASN, + CriteriaCC, + CriteriaIPNetwork, + CriteriaName, + EMailNotificationAddress, + InsideFilterASN, + InsideFilterCC, + InsideFilterFQDN, + InsideFilterIPNetwork, + InsideFilterURL, + ] + # temporarily disabled in the Flask-Admin view + certificate_related_models = [ + CACert, + Cert, + RequestCase, + ] + + def __init__(self, engine): + self.app_config = self.get_config_section() + self.app = Flask(__name__) + self.app.secret_key = self.app_config['app_secret_key'] + self.app.teardown_request(self._teardown_request) + db_session.configure(bind=engine) + self.admin = Admin(self.app, + name=self.app_config['app_name'], + template_mode=self.app_config['template_mode'], + index_view=CustomIndexView( + name='Home', + template='home.html', + url='/')) + self._populate_views() + + def run_app(self): + self.app.run() + + @staticmethod + def _teardown_request(exception=None): + db_session.remove() + + def _populate_views(self): + for model, view, kwargs in self.string_pk_table_models_views: + if kwargs: + self.admin.add_view(view(model, db_session, **kwargs)) + else: + self.admin.add_view(view(model, db_session)) + + +def _get_patched_get_form(original_func): + """ + Patch `get_form()` function, so `hidden_pk` keyword + argument is always set to False. + + Columns with "PRIMARY KEY" constraints are represented + as non-editable hidden input elements, not as editable + forms, when argument `hidden_pk` is True. + """ + def _is_pk_string(model): + inspection = inspect(model) + main_pk = inspection.primary_key[0] + return isinstance(main_pk.type, String) + + def patched_func(model, converter, **kwargs): + if _is_pk_string(model): + kwargs['hidden_pk'] = False + return original_func(model, converter, **kwargs) + return patched_func + + +def _get_exception_message(exc): + """ + Try to get a message from a raised exception. + + Args: + `exc`: + An instance of a raised exception. + + Returns: + Message from exception or default message, as unicode. + """ + if isinstance(exc, FieldValueError): + return exc.public_message + else: + exc_message = getattr(exc, 'message', None) + if exc_message and isinstance(exc_message, basestring): + return as_unicode(exc_message) + return u'Failed to create record.' + + +def _patched_populate_obj(self, obj, name): + """ + Patch original method, in order to: + * Prevent Flask-admin from populating fields with NoneType. + * Append a list of validation errors, if a models' validator + raised an exception (not Flask-Admin's validator), to + highlight invalid field in application's view. + """ + if self.data is not None: + try: + setattr(obj, name, self.data) + except Exception as exc: + invalid_field = getattr(exc, 'invalid_field', None) + if invalid_field and isinstance(self.errors, MutableSequence): + self.errors.append(_get_exception_message(exc)) + raise + + +def monkey_patch_flask_admin(): + setattr(form, 'get_form', _get_patched_get_form(form.get_form)) + setattr(Field, 'populate_obj', _patched_populate_obj) + + +def get_app(): + """ + Configure an SQL engine and return Flask-Admin WSGI application + object. + + Returns: + A flask.app.Flask instance. + """ + with logging_configured(): + monkey_patch_flask_admin() + engine = SQLAuthDBConfigMixin().engine + admin_panel = AdminPanel(engine) + return admin_panel.app + + +if __name__ == '__main__': + # run admin panel on development server + a = get_app() + a.run() diff --git a/N6AdminPanel/n6adminpanel/static/logo.png b/N6AdminPanel/n6adminpanel/static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..9b60233768c104da2dc787976716bb52f42cc9a1 GIT binary patch literal 11724 zcmXYX1zc3!^Y;=`(jeU+@Bk{^tgtNIh;)Z^r*w&QBP9(2(hbrL(vmLHNFyK(0`KAX z{y#3RUiR+2_spF6#!Q5&vJ4Iu1r`JX!I6`NtAlq0_{qgU1Ap5Ty7+=OBv*AA7^HHX zY8U(m%~U}K4taR|liOaL1dd=j$?CX5AWw-Ne~=&MbT>IADfC|$DCj7p%=+rO z;1Ic+l(w6sqrJU_gBwKB#lpnR!ko_2+RchiMovjpD+req0-=M*!6h`j5QhfV+8Q!* z{*JFEIh5*-h8UtmF!F<=^H))o!nxi}U4QOlau`q*x( z3O6|&;Z!G41SBZ118wqO4kHCRuKnaE^U~6;yW6hI6_mGc#myM+v`T%i)=zd;SW^O3 zF|>7%l$FJ;Sn+9a;c!SzO<-%Qg&4CX_F5NKvi$0m^Xhg^kV0yp>vptOqbw>mC7NOe zyb}f!HWWM8^^)6f6L*GhOycyx)3Fzt@VvIEb1TP1*|+MgyE^Zmc==Gm`5mM-CdJOG zUv;8tR-#5mjTwCmgsd)b(Q%Bf&c=W{9AVN{!r`sRv;7xKcJ4VJPw*-BnsM-MCUT>rK(u*K zP*Qcn(DEK2;} z+20-eNusD~VY}f|!u^PKOz5 zEZMi0#WBzL__`<^A(j(AP(KaV8|5TP9W^+yG*8qbMd2u&q~>^fc7 zn!xL7^99NT917(U6Jb^FzRAwfxCp80U>+c(awIc_lAxjpL)72V^i-q8>IpAt&1WJo ziLhNAU7Yjq?C)>NPQtmoIQ^P6SU1B5x_dASk8B8OaP>PK-p6p;k-Z4U`cNL8@Huu4 zW%9=lV zTU%X)(9+U|!x=D8k;D5et&;9^xFmF1B{oSKNuf5MZP~?B6Ss}mMkrbeXc-<-!(Nw zU|K`=^5x6plM@MjeX4;N%8$p#PB@8Ng{{sgB^7-6Pl7C)Nr!rWVl=ycWTcQ`7<-mq zRP+P~2S@PzU#8l+I_K-Z7Rjlpj3OeGq4+~?Wat@-)U>;bJXW`2^r7#oG}zQMHN$Fa zxqp3ge+qFrUX?h$_5}}vijK}GEKDIMCl?+S6;fBnqpE?LuhC=NZTHvQ_`(h$0%t{4 z&O%=HOxbiy!%okAVta;0XBl>WLS4sj?EsXD;Z8`R=G5;78z_Sn;wOSoUj{Lr7k5mmka< zqBPoXXx1+`+Wh|gd#yOfzb8-fqthXRPqs+CzW!!SM^8_ZiLiTk7}MF=8GId?pc7qB zZ|~IPj8vf6mw3*mL7vqTauMG#>H2uU&&vil#MmY=^&-Hcd%Mp%)XQmWie6 zKG_&l)7HkQ(T)ZIil!96&CAQfz`}yLx$&YRBf(%WI$|t$T>_EXmwPu3KMLhW9~v4O zt~TOc$7g599vs-|cluMew6s`RS$X-!LUwGeLY_Q9Qd2`z1P@@COjgm#)Mra`G;rnT z+760Gpwq_AHIBH)^Bz~67e_?x3OU>RR2%In}YEGsTo2iKJQ-N9kT1kYcJ?JApSfe zU+h{3xws%b&<>AAwCiV`r88J;R8-Wi+1Y3G^z=?nPJv}*WtuJ&?nfgSFJ;EY#st03 zST!s4Be2EtrDGsY$bl8~0tVF(?RMQ;PiN;dsJ51grA*E!F2-v305W@Y)BB&gny#)q z6ciM>>ZLUFDRO@l>ViECzLphCSVzFmEML5EGsO@QK`1I?9F{N6@fhL(yu%8(U0iXx zyK&EA|78o|Cud+#Bc{FwDS5c!<26|W{f6b~Nhd(+ZjK zVRu9<2T0K!q%!Dsq3UF)m0cl#lxcQ2fYi=n>kr`D!{xF1d+k_oaLz2p$Is{- z?FvWgYx7@Ml@(ZO+S=X(q+m6>{5GMAx?o>U=jnRYIf>fpbz))=%{_~?lNKTQ5&6xR z$y-HbW#LyU3JSd=uR5b!J&$aBvyqLz(w|q?}DIc=&hd<~C8SzZm`U@>bPE>k*;4wrse%+`NoImFCRM zOgz10SiI&UIsppPQs8YV1g39*Sz6(@X&8zRYqaP2m(Fi*>YnC2Xs+8%qb+5+E}8K6 zm(Xfy@eWF~W4jcs2d}_fWzj^a-_3vR>P8q20P$KV%4};HN9@%O|C#%9NLv z%f=J@siUBCH4JEPuthA{{$T3O39&eHn5rrIa2Is2Eiue-3f|;05T{X9kTubzPM$(`xs%l#= z;+?&{Ql&xboI^5&_Cp-h)UV{+fqHB-0=FJk#J#MB|3>0v09lgcNcJjaRhwy=jBMFk zsdhD}|35EYKEA1(!=GkecR9X^=gbF7a0LYvbg`2qPXs1Rt(sPFa|CQ-9 zvua8T-BE|Fb~h4|n>zL<&l>NeFg|*>W^M(f92G~Z%Oe#ZxBB`?ABXhxt`PKt`H88M zwXoJLX^xUyy7+2NI)NsIP)0}x@+!_Mm6bpxRN+&g?>{b9$?@Eu3lP0VvK67AI$`Yjy*I5+yY4=e4wM3- zl$3WWO9jsy+azwUN;h0eq^KG^Wo5%19$vfV*C$^vOhI)*1m1K1!6D1db0kMGF#OnY zpW&h27&=?7wQn^$TR%da33!X#40O}|wtMOu*C-nd=9Uzc-$jH`O(#m_gNP4#R>8_k zEa-yjb^dnloIBPElGPb?7#kj4m~uY;PM^O`62Zujzj48UN;DWZ=mt?hnz7c2QA8k2 zxsKxkCKD2q5;@*I--xC!R;e@k$dWxdI#K?#rR8A0!Kf1#{iT!|ej|naJ(4kc;D3;N zEMqzd6jFfaqosw|cnHA@oGQ=}LGC3`N^MBU#}Vp zyGwsKf@q@{vjoB+0cewWfus->$X>6+cTK{*;C%OZLKYTPe*Sv0!y~-5D+a&5!6kz? zO~1q6VN%(;MqXdohY0B4khsGnMQTu0KhX;#5PQ$ix+W%$5+-N@p|GSMh`E1rsV@SZ zAoLJBNHR)1a>#Z+;0c$Y>4<>|y@=`HXug+0z9%qrOrV3o-^#ZtM{5J??>x9v|<_nW;=phtKf; z34`SYYOwX+-CmwtZ|Cq#Wq)bd^_m;n+9;UTuM86L1XtLrucywfuErOoQ8@~P-mEu1 zr?OZN|56Ij+l5y^VB>V+wTy=c|NZUCL;lyV@{9~es^{l+n*4Nu(3mBifAOuJyF`n2 zJ@cs-7`ug)N=i!7EaVUkO--9DS)qKffv?k0cxDFY4X>S!4*A~xzVq#cy}703M$?MF zK2rHTlw@|m5`JgD?7WS zfdRFzpI`U(>D0mk1_W>p6O6*~cj~IDfdJ+K2u>HPY~0`74vmiwL=dqUTUvfhd`X6d z{CJ7jIZI7$QQCSgPN!T$q8)dlB3rgsua4%J%#_@cWtpyiry6lOE{hJCb2m3Pix;U& zJ&eesWBqn_C{b zvteOq>l+(piBL)|Q+U&*iDmy21?qGyR$|lhz1g9W5lK6{a-Bb781*aUWs8~4&Nl{H zs$CuQOswG-T|20NSr{a8p7D4KOAI#}eGM`mo3E+BSY2BSs{RC2BVYmxHu!*fRVCgs{Ql1JuD->V0(=Y(MafizW`pu~u;sf-nqIQ&Xq)K1|>9 zS*>rq)|BC|^Qr=^s`rV`{avaXWQ3Bof=eQ~b z27;vya^An6k}wk%f=7I1%Po&yQg-qzW_d#GUc`#UmV(s@6vpsZPV*srFK=&;l{+2< zW<&y`OttT|OF=5-q&&Cg_P|uZY(ag&bLwW#xI5LSCMIWCD3)YpYiCgs5uLZbWchf9OD=kqaO`4#E6OD*CAL`71vSoGm6HY5yh{z#apLE! z1ggKk|B~VBI>G#B&z`M~39RtyT;DAij+IwfQ!CM?onU_=6gTs!(Pf=UKhhC9UU|Pw zoixYJ&rhPMsrh&wS$?UA!&JUhynGRJ%STEGJBaV3>K(3+=3DYlRo7hI=$zEzYCe+T zxcO?tA9DO}Fq>i$n#zf4YFd2`o2WNQfFUk8Hkn?1@9 zXCQCM>!`^5tPFR-P#Gzy^j}DAEZq#6yg-z-xNc|6Em&gWeT{9MN*_9hXJkgc zqV8hPHS_Koi6U`doO$b~_?(aLS?WszZfn16pFcupOxmbZKYsjBTy_rOSd%yxEXD2; zEH$SkQqC1P>qMz0gxp*f_aAt;+uPXmtlVD+B+U=b5II&cWatX`Y{Uovp{M9Uiv)>s zQkdN+)t|`wKlC%ZxViO}>(qIFDoRkwFQxno_BfxldBN^`PP@{s%AV-hQt*wk1muL9+;tosSq*f08 zT3n3XAR&eF@_Q_EdL9w3Zx8Bag;S3C2)R1yH4?bi=KU1NK+6S+2r5*V4;Ta7yXz3$HE`CiwD#zVNczycpq zl+D(oWceWUuK)()%E%1TbY`SySz&$XDZKfa{NEe81lgU~H=fzg?KIKRJec212b}(0 z;EPsE_TTl7sU3>z?)mW4r_JG=M}BEYOJPw_x}rcPpY2?4YyqpVgP)&R>kBH9x$9D! z$owRov=fa6e&Wb}r=|wJk#^|AKi7rD#e$|TyB;nnwUY93ULDRchtmm z#dCKfOf1eP>F5ck9?GWs<(UG7%_1vaP-{uvEiYwVs3i~Y?Dv@>B1;^Tm-L!;zF||# z9~Gx0Pohipn<5BHRUymEXC|Yqo=Bvm!XpZPVyXr>US5;Ktnv!hy-`g9N~=HAyOy(| zLppCHRiHF9Zg)p3#W~$e)T(Us6;(1Dd6LYuPZd6KNO*a9U53MITY3J>z zs9hv-eDNxN_5Z|J4fJm0q;jA)EXfea>(^tK_H}9Xc-@a53QuXBIrn@0{P;d;!t&X<2jUfDXdpD`Sd@{I3{^LlXjopT784PZNQ zQhhKP;KI8t#XC;-Db3<{11Y;0w9i6pG%wKsuJw3oH@#+6kT5t%a1dQ%Hn^VY*dZY&hbhA=X@W&fCoKF;i*0_u&>E6v&M@;uT9x=o zkT@hy730;bSIw=h8{p)HMtjPw(ex0dOn!1cB#SDpFi5>t_{WbQr!2>^<8c!00WSpX zFsa-M+%rDbhwz992{;@T(2ce2ZS)8@vVed9Fa?2f1)d+Eyq9U(x8(#%%t_tT)6+Xy zyBiz5T>Th8ut#=Vpgwuh-P?DKQcM*|QJM8v`bwLR8ytQTTLFzX#OadzIXm7X z#XdPDWi2eW>Uff=twl6VS;Tr#0BSULB>9i@tT)euk-;thLQe^RwF1)6=yFK?hAL zn(hxzNoh`ApI=-Ie5Cac2gVx(z#sfZxthgK`cyR>;Z}R&WMbaDJWnZ}eo;I`Oy@C2 z-$duiGWGCGh=$A?`ru$m5v7w=z8V8ZBw0{SOhvbRQWvQ5hues_*&sV!q05_vJ*GKe45IJ8Rn;d(I zR>}xD2~*IcIygAU0%OFcMJA4>AJ|($1{EP5M>}F4X{31@w7CJvFTg7V=uQ9xh;)<9 z1*-b&%nThn`_s!mOCj<_>wUf1%DnD_EYF{tgmzd*pTU7cob7+JYpC*)FT%yerP{O~ z3+!KGz|O0inm!g6vv`TpH07V8Q7gp+2P21uhLVZ3EQJ#=#%E+87_(yB-ZM^@czr!DFgKZ4ST=x5C~aj`GH@hRJ^j8M$c}{T)9p+qk%*)uqMuEUv}&xx zOZ*CQx58x_Z21~&a==wf82SAO7UsJ@iu%UJVg!YRxF#HIZ52MfG?4#VR)&c_f@I51 zA)cca6&GiTO+FAR%LMcF73Ftajs{%VZA4C$IVlJ~nAZOaX};2+7c7vPx_a>Xx+!oo ztecf$sDucQ+AnDa;%HaOkds{U3$jgB!QKWYq?(qNWv!3WXF&qTwi7f$LP9#ii2AwS zU+adqwvka$k8W#E56s-c;?X&M{`@&8cRzF*LqbrPRkEW!SNupr(DAnVBx9|p@p}IL zb-B8|EhsG=cGIM#!SjXEa&d8`iTUsYCk|z$(U~$vRJI}IdFlvovC)FW1CKf%Xn-dx ztD-Up@|G@jgg!<79P1?|Az^USviDQht)WGFMj@e}97v|2F#2D= zem$xoBos8`1q5tA-@j!gR{WY58Hoj|fX3Umc|Z`) z&CQjRl!${T-`m@BI^9wNqIMm`5TGDP29FL#&_j4 z;8RfqM3{)5KTWyv3-5;w&R|sUR&vw zt9yH2o%KI3lFRTG5IO)Qpb`WYJ|D|v4S8E?>6$v$e z>4(S04n)1`2nTv7Kj7is$-`qynofpygy~lp4yDuIO}XUc zMRCyIKt@KchmwLWCW$2fYF@7$Q1aZ|+|@pp4hM@(c)(*WpHG(dYU9v@3_)ICg4Dok(s$R z4`pXX$88U?ievWxi6oh)8ko-tnm(Z6fDcRC#F8&cJVBmnVA$R0914=GVnA$^M^ZTp zI>9%(@+d=|QuYU-fF>su=re*w?_;MH^e&x%haU95cJKc_7bJFJDV6zX*J?6is`{^ zZEY``a$1JJ;%=HBoEa@8f(lE|$cO?0Gr5~)D?3#4}CftzM&D zH?UD?eN;Oy5Vi|Vjz+2b+@N;+`*%8_+gaDtgq|}YP*Qby@Y8H=ICf&YcWSC;vE1t4 zbHK!Q5Awx)F2rA^G71Qg?$1?7KHT4d-8RH)r~VrG7RV>>568YByAWUzb@Oj zUI#!y72@={u)_wK2589Y>Gfk0ul0Vcao^YccRdj>cf%niC3Uq^5T~lG9d6Pa#ueMg z$H%ALVDk)@S{M~z_tn)E=gNO6@*GATRHdb*6kN1*b#(yYgQ}}JAzdJ!@JUF*K*jzn zO!S#m7R)11KF$KTV>LM}^#H;W|M@d^x?)&(IL2tE0G?K*es_^Vs;ZV2db;8O(My=nPH_GJ@fOVr(0u^>gt3f9NK{ZGT>5DT<$3&zqLDjg>Y#^F|@R_ zz8ruZ4mnGpg)G!t6Ls9**!~~gA>*?lCemH&4nfy$ae0nQ3&nPHbOh`d0s)Hn7~lzv zAz>A~g=uqyb#}hCv9&$9Saf6r@J9N|5U#GCI9C`CT6tIVo%bJrOzVC0r)E0>9{ly& zyb^>yoh|zubuTVb{`vDqvDIOvgE|zS*+d4IcO4J+KG6UhK+=V1b4ZJeL#A;-!gc{5 z4f^_(E%f6@o}5>PZDgSS^k(qcI{mwH?i(Dm6-;vc)l9J1!hFb zWSAuzKd2KN;Oej1j*&q}4z!&nLDn+=Co0Y&aCvbd z?c~Jiy7^&Z+w5q$Jx$o1ZQ1u!2_HtI^CCt5`M-bvCW}>Q?UvifeJ&0_7nAkHiw{jr z0szw1j*jB4uDpCNep*5TfddHO0Z^?xpvq+x75hN+ArO#tkFfsc9wgmkPx1pS=#d*m zk-h@J&ieiP_v&|J6xw=vW}u`$wl0|!(|{Q*W4jr7`=|Bj7$isc+}uS1wAueoSUFo1 z6SB}?OKNInMxP*CZTFMgZm|)ESV{u0QP2Pd_RjJ7dH+I<*~!YoJt-t>VE0#xD_y8D z`0lIw+am&U@`(T5d@%;)ebn#o-@mK7yAgm&0;a^k%}vEi^V!+i$E#H6HQ`fHMFSso9S~g;<{%;0t=!yP$co=3 zHK?z$uuLC{Pe?%_|I(;K5hPcFe9`8` z;bLEJZwNkY2)Io{&CVNXD-{N!+QytDt_NZZ@LXmQ^sp%0Ch{D_+X=nVG? zxK=`9BB9G#&y)2(f5y%0O3TU^eW5wvw!vh8oV+~No58LwaTj2pgKGe)#Kg!bX-f+{ zp0u{pagoQH78DUl0{#sV4CDHI-bl{Qjssi(w2&jp^;;6JmE7I=h9@R0Z=3*c+yqAB zCg=#vvn~z~N4)=1Qo`y29HSqTlj6YemQzzBaCUN{3&nQKU8X6k-FpT$r6sFz$ zK{Q0!EmTWvlVxz4AYb3?SEch;mXy%J^5TcRlhV`W(C}y!0pf&fussjO2NJV09@sfN z761`lz%B*%uA`$v;jU9}MIaP#-zMEj2z&e}2(*&nVS94I<0 zw+%J#%0LozKF-WIsf8BsJ4Gw)5*HU2(6qqRR8o-doSuL5V`dqdm_n!2 z5PPNp*CR%&zkh#7OCtk_$iTxxG&MElwOf>GY+>=?QEP&511SGIaJTg9m#UuLNSh-l z*PFvhe3d7sr;>p7^SCz5KLcY}CtkoC_1ezzeB2ShMqb-3&KUqS2{7W1vt?R5^8*6| z!TGHEtsXd0Brkq=c^KraA(nYqS!MS}mm8H>BYQ^EdANY`C@d_@Ei6Rs{fKvTtWPDS zW@TsB2Nnq%;^gjb5ln0VMg&`q629AutR;jMa2?Blq@fK+Vl-X(6|rD@TIh zw1YmzDa9pnZUXWR@UjPej@pZ-UIK{per1L1_o}|O_J?o@sG^TGACRW4*2iu&HC=Xg zw%1;1F{k^UM!8`-1qwQDU)=;W3e=7DR3m>R@sH-2uLJHGHCr!)6>hsl0E!C=3-?S+ z-~kg8KLSpH@(HYHhc7wcy$j9GfXA|}Zfya&``aGf04hKh4a@posC9L0F)18@~KD_e*kWn}ch78DznyM-!ARqupL^@zwtB9VUPQQHr{(W@z-2E~r zZ)_>+RRMSGe^xptc2sSsb(fUEAQAOrO)GnC{s7dRJUs`4pF9=EFp=eZH%f*Nqa%sp^1X5b zxcrEDSAe+@fTso2%uO2)%$9(S_81gUNp#W66`6alPBtFD%=BcvpA70N2%&ra{0W%1 zd6W_zcNcnRM=P}KT9v5us30Y}!G#{N>EKrj5unD=%S5!aagXbm<99(0C=DcC5zWWP zr~O6g-u@#fEVX)qbRtFeX-T^6|+)CDBkOT_&%3^qyX^TpTq?K z=}ZqcI9RwoK-qxujs#HyQ$Gw`Tm+BV@op^Rk?t^Zb0_M=xOFct(|CAz0M|^U<9dq{ zNOrncukg6j%X@21i(lVi4#v@1BAx@x1d>xR-~iF^6wLf+*O)%>I7EB^br_Vu?q;~F zr>6`c7N^k1FI{^Cc7XbiEw_)j)IETgkgF*$Bp5GLn^2$tg%bPoO+IU64Y97C9?u#o zzda%}?)5zxNSi*et(~_<$z#%;fUu`v{~Q;G2V#o^0isp{q|{uc!R?T1GzA~lBi{me z_Slg~Na)tCb@1;t=yc6JRc*aBFlMu(+HQbVtxT`;;iz6R=A zbHD@iRG%;+l)grHQ%&#g(F5DvE#)FmOyfiXy!=C4T-^BV=-3zzc-&l8ECAxsLWUe9 zX5hXc5Ij6Qpodns#xfuG$K$$5_j!ONP*ueTGFx6={!wP*Q1To9thHFEv&6%C`V?uh zGy=zr? zfX1_oerFVL#|L<=v7=)g7{2oTJIo^Jbxa2k6%eb{_4VMfOo5G~fCrIAyPx@WFDol6 zAzhm(3ZS3`+@I1u@(n=GAc0_@j9ch0xJk2xy2eIn6BBx{zBMMjXyv*MSZ|7zKfoDG zm@2@i)?;fLFtUD7pCSO3?u&7GP<+p=xSEAdD&1ZlTH4qTlOqPVCrPR#Gh_4jzs2rN zcTgGp-t$H?aE1#ju$4@B+8>E{pIL#FQDE%)3bpX~eJxy;1HCIPQ=u)cG}!GNtWdB# zTu}p^N>I(d<^DH~YL<6p6Fj004`L1m!nO|=E1=u-6bgl=2|68bE-<%J$)$a +{% endblock %} diff --git a/N6AdminPanel/setup.py b/N6AdminPanel/setup.py new file mode 100644 index 0000000..70f92ad --- /dev/null +++ b/N6AdminPanel/setup.py @@ -0,0 +1,34 @@ +from setuptools import setup, find_packages + +requires = [ + 'n6lib', + + 'Flask==1.0.2', + 'Flask-Admin==1.5.1', + 'SQLAlchemy==0.9.10', + 'WTForms==2.1', +] + +setup( + name='n6adminpanel', + version='2.0.0', + + packages=find_packages(), + include_package_data=True, + zip_safe=False, + install_requires=requires, + + description='The *n6* admin panel web application', + url='https://github.com/CERT-Polska/n6', + maintainer='CERT Polska', + maintainer_email='n6@cert.pl', + classifiers=[ + 'License :: OSI Approved :: GNU Affero General Public License v3', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + "Framework :: Flask", + 'Topic :: Security', + ], + keywords='n6 network incident exchange admin panel', +) diff --git a/N6Core/MANIFEST.in b/N6Core/MANIFEST.in new file mode 100644 index 0000000..144ee55 --- /dev/null +++ b/N6Core/MANIFEST.in @@ -0,0 +1,3 @@ +include *.txt *.rst requirements +recursive-include n6 *--data +recursive-include n6/data * diff --git a/N6Core/console_scripts b/N6Core/console_scripts new file mode 100644 index 0000000..4d8fec4 --- /dev/null +++ b/N6Core/console_scripts @@ -0,0 +1,6 @@ +n6archiveraw = n6.archiver.archive_raw:main +n6aggregator = n6.utils.aggregator:main +n6enrich = n6.utils.enrich:main +n6comparator = n6.utils.comparator:main +n6filter = n6.utils.filter:main +n6recorder = n6.archiver.recorder:main diff --git a/N6Core/console_scripts-nonpub b/N6Core/console_scripts-nonpub new file mode 100644 index 0000000..2f399af --- /dev/null +++ b/N6Core/console_scripts-nonpub @@ -0,0 +1,8 @@ +n6digest = n6.archiver.digest:main +n6anonymizer = n6.utils.anonymizer:main +n6notifier = n6.utils.notifier:main +n6counter = n6.utils.counter:main +n6manage = n6.utils.management.n6manage:main +n6stream_test_api_generator = n6.utils.stream_test_api_generator:main +n6splunk_emitter = n6.utils.splunk_emitter:main +n6notifier_templates_renderer = n6.utils.notifier_templates_renderer:main diff --git a/N6Core/n6/__init__.py b/N6Core/n6/__init__.py new file mode 100644 index 0000000..e25ce53 --- /dev/null +++ b/N6Core/n6/__init__.py @@ -0,0 +1,3 @@ +from n6lib.common_helpers import provide_surrogateescape + +provide_surrogateescape() diff --git a/N6Core/n6/archiver/__init__.py b/N6Core/n6/archiver/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/N6Core/n6/archiver/archive_raw.py b/N6Core/n6/archiver/archive_raw.py new file mode 100644 index 0000000..39177c1 --- /dev/null +++ b/N6Core/n6/archiver/archive_raw.py @@ -0,0 +1,948 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Component archive_raw -- adds raw data to the archive database (MongoDB). +A new source is added as a new collection. +""" + +import datetime +import hashlib +import itertools +import math +import os +import socket +import subprocess +import sys +import tempfile +import time +import re + +import gridfs +import pymongo +from gridfs import GridFS +from bson.json_util import loads +from bson.json_util import dumps + +from n6lib.config import Config +from n6.base.queue import QueuedBase, n6QueueProcessingException +from n6lib.log_helpers import get_logger, logging_configured + + +LOGGER = get_logger(__name__) + +FORBIDDEN_DB_NAME_CHAR = '/\\." \n\t\r' +FORBIDDEN_COLLECTION_NAME_CHAR = '$ \n\t\r' +INSUFFICIENT_DISK_SPACE_CODE = 17035 + +first_letter_collection_name = re.compile("^(?!system)[a-z_].*", re.UNICODE) + + +def backup_msg(fname, collection, msg, header): + with open(fname, 'w') as f: + if isinstance(msg, basestring): + payload = (msg.encode('utf-8') if isinstance(msg, unicode) + else msg) + else: + payload = (repr(msg).encode('utf-8') if isinstance(repr(msg), unicode) + else repr(msg)) + + hdr = (repr(header).encode('utf-8') if isinstance(repr(header), unicode) + else repr(header)) + f.write('\n'.join(( collection, hdr, payload ))) + + +def timeit(method): + def timed(*args, **kw): + start = datetime.datetime.now() + result = method(*args, **kw) + stop = datetime.datetime.now() + delta = stop - start + print '%r %r (%r, %r) %r ' % \ + (str(datetime.datetime.now()), method.__name__, args, kw, str(delta)) + return result + return timed + + +def safe_mongocall(call): + def _safe_mongocall(*args, **kwargs): + count_try_connection = 86400 # 5 days + while True: + try: + return call(*args, **kwargs) + except pymongo.errors.AutoReconnect: + LOGGER.error("Cannot connect to mongodb. Retrying...") + time.sleep(5) + count_try_connection -= 1 + ob = args[0] + if (isinstance(ob, JsonStream) or + isinstance(ob, FileGridfs) or + isinstance(ob, BlackListCompacter)): + LOGGER.debug("backup_msg") + try: + backup_msg(ob.dbm.backup_msg, ob.dbm.currcoll, ob.data, ob.headers) + except Exception as exc: + LOGGER.debug('backup_msg_error: %r', exc) + ob.dbm.get_connection() + elif isinstance(ob, DbManager): + LOGGER.debug("backup_msg") + try: + backup_msg(ob.backup_msg, ob.currcoll, ob.backup_msg_data, ob.backup_msg_headers) + except Exception as exc: + LOGGER.error('backup_msg_error: %r', exc) + ob.get_connection() + if count_try_connection < 1: + LOGGER.error("Could not connect to mongodb. Exiting...") + sys.exit(1) + return _safe_mongocall + + +class IndexesStore(object): + _collections_tmp_store = {} + + def __init__(self, connection, db_name, collection_name): + self.db = connection[db_name] + collection = self.db[collection_name] + docs = collection.find().sort("ns", pymongo.ASCENDING) + for i in docs: + coll = Collection(i['ns'].replace(''.join((db_name, '.')), '')) + self.add_to_storage(coll, i['key'].keys()[0]) + + @staticmethod + def add_to_storage(collection, index): + if collection.name not in IndexesStore._collections_tmp_store.keys(): + # new collection, add index, and initialize key in store dict + collection.indexes.append(index) + IndexesStore._collections_tmp_store.update({collection.name: collection}) + else: + # collection in store, add only new index name + IndexesStore._collections_tmp_store[collection.name].indexes.append(index) + + @staticmethod + def name_of_indexed_collection_n6(): + # simple select a collection, no system and no tip chunks + # to check the amount of indexes + return [name for name in IndexesStore._collections_tmp_store.keys() + if ('.chunks' not in name) and name not in ('n6.system.namespaces')] + + @staticmethod + def cleanup_store(): + IndexesStore._collections_tmp_store = {} + + +class Collection(object): + __slots__ = ['name', 'indexes'] + + def __init__(self, name): + self.name = name + self.indexes = [] + + +class DbManager(object): + """""" + + def __init__(self, config=None): + """ + Args: + config: dict containing: mongohost, mongoport, mongodb, + count_try_connection, time_sleep_between_try_connect, uri + """ + if config is None: + config = Config(required={"archiveraw": ("mongohost", + "mongoport", + "mongodb", + "count_try_connection", + "time_sleep_between_try_connect", + "uri")}) + self.config = config["archiveraw"] + else: + self.config = config + self.host = self.config['mongohost'] + self.port = int(self.config['mongoport']) + self._currdb = self.config['mongodb'] + self.uri = self.config["uri"] + self.connection = None + self._currcoll = None + self.conn_gridfs = None + self.time_sleep_between_try_connect = int(self.config['time_sleep_between_try_connect']) + self.count_try_connection = int(self.config['count_try_connection']) + self.indexes_store = [] + self.backup_msg = '.backup_msg' + self.backup_msg_data = None + self.backup_msg_headers = None + + def get_connection(self): + """ + Get a connection to MongoDB. + Try `self.count_try_connection` times, then (if not succeeded) + raise SystemExit. + + Returns: + `self.connection`, as returned by MongoClient(, ). + + Raises: + SystemExit + """ + count_try_connection = self.count_try_connection + while True: + try: + #self.connection = pymongo.mongo_client.MongoClient(self.host, port=self.port) + self.connection = pymongo.mongo_client.MongoClient(self.uri, + sockettimeoutms=2000, + connecttimeoutms=2000, + waitqueuetimeoutms=2000, + ) + return self.connection + + except pymongo.errors.ConnectionFailure: + LOGGER.error("Cannot connect to mongodb@ %s:%s. Retrying...", + self.host, self.port) + time.sleep(self.time_sleep_between_try_connect) + count_try_connection -= 1 + if count_try_connection < 1: + LOGGER.error("Cannot connect to mongodb@ %s:%s. Exiting...", + self.host, self.port) + sys.exit(1) + + @safe_mongocall + def get_conn_db(self): + """Get connection to db.""" + return self.connection[self.currdb] + + @safe_mongocall + def get_conn_collection(self, gridfs=False): + """Get connection to collection.""" + return self.get_conn_db()[self.currcoll] + + @safe_mongocall + def get_conn_gridfs(self): + """Get connection to gridfs api to put, and get files.""" + assert self.currcoll, 'not set self.currcoll' + self.conn_gridfs = gridfs.GridFS(self.get_conn_db(), collection=self.currcoll) + + @safe_mongocall + def put_file_to_db(self, data, **kwargs): + """Put file in mongo.""" + assert self.conn_gridfs, 'not set self.conn_gridfs' + return self.conn_gridfs.put(data, **kwargs) + + @safe_mongocall + def get_file_from_db(self, id_): + """Get file from db.""" + assert self.conn_gridfs, 'not set self.conn_gridfs' + return str(self.conn_gridfs.get(id_).read()) + + @safe_mongocall + def get_file_from_db_raw(self, id_): + """Get file from db, raw not str.""" + assert self.conn_gridfs, 'not set self.conn_gridfs' + return self.conn_gridfs.get(id_).read() + + @property + def currdb(self): + return self._currdb + + @currdb.setter + def currdb(self, value): + value_str = str(value) + if len(value_str) >= 64 or len(value_str) < 1: + LOGGER.error('to long db name in mongo, max 63 chars, min 1 char : %r', value_str) + raise n6QueueProcessingException("to long db name in mongo, max 63 chars, min 1 char" + ": {0}".format(value_str)) + for forbidden_char in FORBIDDEN_DB_NAME_CHAR: + if forbidden_char in value_str: + LOGGER.error('name of db: %r, contains forbidden_char: %r', value_str, + forbidden_char) + raise n6QueueProcessingException("name of db: {}, " + "contains forbidden_char: {}".format(value_str, forbidden_char)) + self._currdb = value + + @property + def currcoll(self): + return self._currcoll + + @currcoll.setter + def currcoll(self, value): + if value is None: + self._currcoll = value + return + value_str = str(value) + m = re.match(first_letter_collection_name, value_str) + if not m or len(value_str) < 1: + raise n6QueueProcessingException('Collection names should begin with an underscore ' + 'or a letter character, and not be an empty string ' + '(e.g. ""), and not begin with the system. prefix. ' + '(Reserved for internal use.)') + for forbidden_char in FORBIDDEN_COLLECTION_NAME_CHAR: + if forbidden_char in value_str: + LOGGER.error('name of collection: %r, contains forbidden_char: %r', value_str, + forbidden_char) + raise n6QueueProcessingException("name of collection: {0}, " + "contains forbidden_char: {1}". + format(value_str, forbidden_char)) + self._currcoll = value + + def database_exists(self): + """Check if the database exists on the server.""" + return self.currdb in self.connection.database_names() + + def collection_exists(self): + """Check if the collection exists in the database. + Not very good in terms of performance!.""" + if self.currcoll not in self.get_conn_db().collection_names(): + # only for manageApi + return self.currcoll + '.files' in self.get_conn_db().collection_names() + return self.currcoll in self.get_conn_db().collection_names() + + def initialize_index_store(self): + if self.connection: + IndexesStore.cleanup_store() + index_store = IndexesStore(self.connection, self.currdb, 'system.indexes') + self.indexes_store = index_store.name_of_indexed_collection_n6() + else: + LOGGER.error('No connection to initialize index store') + + +class MongoConnection(object): + """ + MongoConnection - a set of common attributes of classes + (JsonStream, FileGridfs, BlackListCompacter). + + Args: + `dbmanager` : object DbManager type. + `properties` : properties from AMQP. Required for the next processing + `**kwargs` : (dict with additional data) + """ + indexes_common = ['rid', 'received', 'md5'] + + def __init__(self, dbmanager=None, properties=None, **kwargs): + self.dbm = dbmanager + self.data = {} + self.raw = None + self.content_type = None + self.headers = {} + if properties: + if properties.headers: + self.headers = properties.headers.copy() + if "meta" in properties.headers: + self.headers['meta'].update(properties.headers['meta']) + else: + # empty meta, add key meta, adds key meta + # another data such. rid, received, contentType.... + self.headers["meta"] = {} + LOGGER.debug('No "meta" in headers: %r', properties.headers) + else: + # empty headers, add key meta, adds key + # meta another data such. rid, received, contentType.... + self.headers["meta"] = {} + LOGGER.debug('Empty headers: %r', properties.headers) + + if properties.type in ('file', 'blacklist'): + # content_type required fo type file and blacklist + try: + self.headers['meta'].update({'contentType': properties.content_type}) + except AttributeError as exc: + LOGGER.error('No "content_type" in properties: %r', properties.headers) + raise + # always add + self.headers['meta'].update({'rid': properties.message_id, + 'received': self.get_time_created(properties.timestamp)}) + else: + # empty properties, it is very bad + raise n6QueueProcessingException("empty properties, it is very bad" + ": {0}".format(properties)) + + def get_time_created(self, ts): + try: + return datetime.datetime.utcfromtimestamp(ts) + except TypeError as exc: + LOGGER.error("Bad type timestamp: %r, exc: %r, collection: %r", ts, exc, + self.dbm.currcoll) + raise + + def create_indexes(self, coll): + """Create indexes on new collection.""" + for idx in MongoConnection.indexes_common: + LOGGER.info("Create indexes: %r on collection: %r", idx, coll.name) + coll.create_index(idx) + # refresh indexes store + self.dbm.initialize_index_store() + + +class JsonStream(MongoConnection): + """ + This class is responsible for the different types of writing to mongo. + This class extJson|Json format stores only + (http://docs.mongodb.org/manual/reference/mongodb-extended-json/). + JsonStream inherits from the MongoConnection. + """ + def preparations_data(self, data): + """ + Data preparation. + + Args: + `data` : data from AMQP. + + Raises: + `n6QueueProcessingException` when except processing data. + """ + try: + self.raw = loads(data) + # calculate md5, inplace its fastest + self.headers['meta'].update({ + 'md5': hashlib.md5(dumps(self.raw, sort_keys=True)).hexdigest()}) + + except Exception as exc: + LOGGER.error('exception when processing: %r %r %r (%r)', + self.dbm.currdb, self.dbm.currcoll, data, exc) + raise + + else: + self.write() + + @safe_mongocall + def write(self): + """ + Write data to db as json store. + + Raises: + `UnicodeDecodeError` when collection name or the database name is not allowed + `pymongo.errors.AutoReconnect` when problem with connection to mongo. + `n6QueueProcessingException` if catch other exception. + """ + LOGGER.debug('Stream inserting...') + LOGGER.debug('HEADER: %r', self.headers) + self.data['data'] = self.raw + self.data['uploadDate'] = datetime.datetime.utcfromtimestamp(time.time()) + self.data.update(self.headers['meta']) + + # for backup msg + self.dbm.backup_msg_data = self.data + self.dbm.backup_msg_headers = self.headers + + try: + try: + if self.dbm.currcoll not in self.dbm.indexes_store: + self.create_indexes(self.dbm.get_conn_collection()) + + self.dbm.get_conn_collection().insert(self.data) + except pymongo.errors.OperationFailure as exc: + if exc.code == INSUFFICIENT_DISK_SPACE_CODE: + sys.exit(repr(exc)) + raise + except pymongo.errors.AutoReconnect as exc: + LOGGER.error('%r', exc) + raise + except UnicodeDecodeError as exc: + LOGGER.error("collection name or the database name is not allowed: %r, %r, %r", + self.dbm.currdb, self.dbm.currcoll, exc) + raise + except Exception as exc: + LOGGER.error('save data in mongodb FAILED, header: %r , exception: %r', + self.headers, exc) + raise n6QueueProcessingException('save data in mongob FAILED') + else: + LOGGER.debug('Insert done.') + + def gen_md5(self, data): + """Generate md5 hash In the data field.""" + return hashlib.md5(dumps(data, sort_keys=True)).hexdigest() + + +class FileGridfs(MongoConnection): + """ + This class is responsible for the different types of writing to mongo. + This class files and other binary format stores. + FileGridfs inherits from the MongoConnection. + """ + def preparations_data(self, data): + """ + Data preparation. + + Args: + `data` : data from AMQP. + + Raises: + `n6QueueProcessingException` when except processing data. + """ + + try: + self.data = data + except Exception as exc: + LOGGER.error('exception when processing: %r %r %r (%r)', + self.dbm.currdb, self.dbm.currcoll, data, exc) + raise + else: + self.write() + + @safe_mongocall + def write(self): + """ + Write data to db as GridFS store. + + Raises: + `UnicodeDecodeError` when collection name or the database name is not allowed. + `pymongo.errors.AutoReconnect` when problem with connection to mongo. + `n6QueueProcessingException` if catch other exception. + """ + LOGGER.debug('Binary inserting...') + LOGGER.debug('HEADER: %r', self.headers) + + # for backup msg + self.dbm.backup_msg_data = self.data + self.dbm.backup_msg_headers = self.headers + + try: + try: + self.dbm.get_conn_gridfs() + coll = self.dbm.get_conn_collection().files + if coll.name not in self.dbm.indexes_store: + self.create_indexes(coll) + self.dbm.put_file_to_db(self.data, **self.headers['meta']) + except pymongo.errors.OperationFailure as exc: + if exc.code == INSUFFICIENT_DISK_SPACE_CODE: + sys.exit(repr(exc)) + raise + except pymongo.errors.AutoReconnect as exc: + LOGGER.error('%r', exc) + raise + except UnicodeDecodeError as exc: + LOGGER.error("collection name or the database name is not allowed: %r, %r, %r", + self.dbm.currdb, self.dbm.currcoll, exc) + raise + except Exception as exc: + LOGGER.error('save data in mongodb FAILED, header: %r , exception: %r', + self.headers, exc) + raise n6QueueProcessingException('save data in mongob FAILED') + else: + LOGGER.debug('Saving data, with meta key, done') + + def get_file(self, currdb, currcoll, **kw): + """Get file/s from mongo gridfs system. Not implemented.""" + pass + + +class DBarchiver(QueuedBase): + """ Archive data """ + input_queue = {"exchange": "raw", + "exchange_type": "topic", + "queue_name": "dba", + "binding_keys": ["#"] + } + + def __init__(self, *args, **kwargs): + self.manager = DbManager() + self.connectdb = self.manager.get_connection() + self.manager.initialize_index_store() # after call get_connection + self.connectdb.secondary_acceptable_latency_ms = 5000 # max latency for ping + super(DBarchiver, self).__init__(*args, **kwargs) + + __count = itertools.count(1) + __tf = [] + def input_callback(self, routing_key, body, properties): + #t0 = time.time() + #try: + """ + Channel callback method. + + Args: + `routing_key` : routing_key from AMQP. + `body` : message body from AMQP. + `properties` : properties from AMQP. Required for the next processing + + Raises: + `n6QueueProcessingException`: + From JsonStream/FileGridfs or when message type is unknown. + Other exceptions (e.g. pymongo.errors.DuplicateKeyError). + """ + # Headers required for the next processing + if properties.headers is None: + properties.headers = {} + + # Suspend writing to Mongo if header is set to False + try: + writing = properties.headers['write_to_mongo'] + except KeyError: + writing = True + + LOGGER.debug("Received properties :%r", properties) + LOGGER.debug("Received headers :%r", properties.headers) + # set collection name + self.manager.currcoll = routing_key + type_ = properties.type + payload = (body.encode('utf-8') if isinstance(body, unicode) + else body) + + # Add to archive + if writing: + if type_ == 'stream': + s = JsonStream(dbmanager=self.manager, properties=properties) + s.preparations_data(payload) + elif type_ == 'file': + s = FileGridfs(dbmanager=self.manager, properties=properties) + s.preparations_data(payload) + elif type_ == 'blacklist': + s = BlackListCompacter(dbmanager=self.manager, properties=properties) + s.preparations_data(payload) + s.start() + else: + raise n6QueueProcessingException( + "Unknown message type: {0}, source: {1}".format(type_, routing_key)) + #finally: + # self.__tf.append(time.time() - t0) + # if next(self.__count) % 5000 == 0: + # try: + # LOGGER.critical('ARCHIVE-RAW INPUT CALLBACK TIMES: min %s, avg %s', + # min(tf), + # math.fsum(tf) / len(tf)) + # finally: + # del tf[:] + + +class BlackListCompacter(MongoConnection): + """ + Performs a diff of a record (patches) to the database, the differences recovers file ORIGINAL + (saves space) + """ + generate_all_file = False + init = 1 + period = 14 + + def __init__(self, dbmanager=None, properties=None): + LOGGER.debug('run blacklist : collection: %r', + dbmanager.currcoll) + super(BlackListCompacter, self).__init__(dbmanager=dbmanager, + properties=properties, + ) + self.list_tmp_files = [] + self.prefix = '.csv_' + self.suffix = 'bl-' + self.marker_db_init = 0 + self.marker_db_diff = 1 + self.prev_id = None + self.file_init = None + self.payload = None + self.dbm = dbmanager + # for backup msg + self.dbm.backup_msg_data = self.data + self.dbm.backup_msg_headers = self.headers + try: + self.dbm.get_conn_gridfs() + # name collection in gridfs is src.subsrc.files|chunks + self.collection = self.dbm.get_conn_collection().files + except UnicodeDecodeError as exc: + LOGGER.error("collection name or the database name is not allowed: %r, %r", + self.dbm.currcoll, exc) + raise + # create indexes + if self.collection.name not in self.dbm.indexes_store: + self.create_indexes(self.collection) + + def preparations_data(self, data): + """ + Data preparation. + + Args: + `data` : data from AMQP. + + Raises: + `n6QueueProcessingException` when except processing data. + """ + + try: + self.payload = data + self.init_files() + except Exception as exc: + LOGGER.error('exception when processing: %r %r %r (%r)', + self.dbm.currdb, self.dbm.currcoll, data, exc) + raise + + def init_files(self): + """Init all tmp files""" + self.tempfilefd_file_init, self.tempfile_file_init = tempfile.mkstemp(self.prefix, + self.suffix) + self.tempfilefd_file, self.tempfile_file = tempfile.mkstemp(self.prefix, + self.suffix) + self.tempfilefd_patch_all, self.tempfile_patch_all = tempfile.mkstemp(self.prefix, + self.suffix) + self.tempfilefd_patch, self.tempfile_patch = tempfile.mkstemp(self.prefix, self.suffix) + self.tempfilefd_patch_tmp, self.tempfile_patch_tmp = tempfile.mkstemp(self.prefix, + self.suffix) + self.tempfilefd_patch_u, self.tempfile_patch_u = tempfile.mkstemp(self.prefix, + self.suffix) + (self.tempfilefd_file_recovery_0, + self.tempfile_file_recovery_0) = tempfile.mkstemp(self.prefix, self.suffix) + self.tempfilefd_ver, self.tempfile_ver = tempfile.mkstemp(self.prefix, self.suffix) + + self.list_tmp_files.append((self.tempfilefd_file_init, self.tempfile_file_init)) + self.list_tmp_files.append((self.tempfilefd_file, self.tempfile_file)) + self.list_tmp_files.append((self.tempfilefd_patch_all, self.tempfile_patch_all)) + self.list_tmp_files.append((self.tempfilefd_patch_tmp, self.tempfile_patch_tmp)) + self.list_tmp_files.append((self.tempfilefd_patch_u, self.tempfile_patch_u)) + self.list_tmp_files.append((self.tempfilefd_file_recovery_0, + self.tempfile_file_recovery_0)) + self.list_tmp_files.append((self.tempfilefd_patch, self.tempfile_patch)) + self.list_tmp_files.append((self.tempfilefd_ver, self.tempfile_ver)) + + # save orig init file + with open(self.tempfile_file_init, 'w') as fid: + LOGGER.debug('WTF: %r', type(self.payload)) + fid.write(self.payload) + self.file_init = self.tempfile_file_init + + for fd, fn in self.list_tmp_files: + os.close(fd) + os.chmod(fn, 0644) + LOGGER.debug('run blacklist init tmp files') + + @safe_mongocall + def save_file_in_db(self, marker, data): + """ + Save file in DB + + Args: `marker` int, 0 - init file, 1,2,...,self.period - diff files + `data` file + + Return: None + + Raises: + `pymongo.errors.AutoReconnect` when problem with connection to mongo. + `n6QueueProcessingException` if catch other exception. + """ + # marker indicates the beginning of a sequence of file patch, length period. + # override these attr, results in a new sequence differences(diffs) + # override is very important ! + self.headers["meta"]["marker"] = marker + self.headers["meta"]["prev_id"] = self.prev_id + # for bakup_msg + self.data = data + self.dbm.backup_msg_data = self.data + self.dbm.backup_msg_headers = self.headers + try: + try: + self.dbm.put_file_to_db(data, **self.headers["meta"]) + except pymongo.errors.OperationFailure as exc: + if exc.code == INSUFFICIENT_DISK_SPACE_CODE: + sys.exit(repr(exc)) + raise + except pymongo.errors.AutoReconnect as exc: + LOGGER.error('%r', exc) + raise + except Exception as exc: + LOGGER.error('save file in mongodb FAILED, header: %r , exception: %r', + self.headers, exc) + raise n6QueueProcessingException('save file in mongob FAILED') + else: + LOGGER.debug('save file in db marker: %r', marker) + + @safe_mongocall + def get_patches(self): + """ + Get patch from DB + + Args: None + + Return: first_file_id, cursor(with patch files without first init file) + """ + cursor = self.collection.find( + { + "marker": self.marker_db_init} + ).sort("received", pymongo.DESCENDING).limit(1) + + row = cursor.next() + date = row["received"] + first_file_id = row["_id"] + + cursor = self.collection.find( + { + "marker": {"$gte": self.marker_db_init}, + "received": {"$gte": date} + } + ).sort("received", pymongo.ASCENDING) + + LOGGER.debug('first_file_id :%s date: %s', first_file_id, date) + return first_file_id, cursor + + def save_diff_in_db(self, files): + """ + Saves Diff, used Unix features: diff. + + Args: `files` + + Return: None + """ + file1, file2 = files + f_sout = open(self.tempfile_patch_u, "w") + if BlackListCompacter.init: + BlackListCompacter.init = 0 + subprocess.call("diff -u " + file1 + " " + file2, + stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + f_sout.close() + + self.save_file_in_db(self.marker_db_init, + open(self.tempfile_patch_u, 'r').read()) + LOGGER.debug(' marker init in db:%s ', self.marker_db_init) + else: + subprocess.call("diff -u " + file1 + " " + + file2, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + f_sout.close() + + self.save_file_in_db(self.marker_db_diff, + open(self.tempfile_patch_u, 'r').read()) + LOGGER.debug('marker in period in db :%s ', self.marker_db_diff) + + def generate_orig_file(self, cursor, file_id): + """ + Generates one or more files, patching one file to another. + Used Unix features: patch. + + Args: `cursor`: (with all the patch from one period) + `file_id`: first init file id to generate first patch + + Return: None + """ + LOGGER.debug('BlackListCompacter.GENERATE_ALL_FILE: %r', + BlackListCompacter.generate_all_file) + # generate first file + files_count = 1 + # stdout in file + f_sout = open(self.tempfile_patch_u, "w") + # first diff file post init in GENERATE_ALL_FILE mode + if cursor.count() > 0 and BlackListCompacter.generate_all_file: + out = subprocess.call("patch " + self.tempfile_file + " -i " + + self.tempfile_patch_tmp + " -o " + + self.tempfile_ver + str(files_count - 1), + stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + LOGGER.debug('patch_next_file(return code): %r', out) + self.list_tmp_files.append((self.tempfile_ver, + self.tempfile_ver + + str(files_count - 1))) + os.chmod(self.tempfile_ver + str(files_count - 1), 0644) + + # # first diff file post init in GENERATE_ONE_FILE mode + elif cursor.count() > 0 and (not BlackListCompacter.generate_all_file): + out = subprocess.call("patch " + self.tempfile_file + " -i " + + self.tempfile_patch_tmp + " -o " + + self.tempfile_file_recovery_0, + stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + LOGGER.debug('patch_next_file(return code): %r', out) + + else: + file_db = self.dbm.get_file_from_db_raw(file_id) + patch_file = open(self.tempfile_patch_tmp, 'w') + patch_file.write(file_db) + patch_file.close() + + out = subprocess.call("patch " + + self.tempfile_file_recovery_0 + " -i " + + self.tempfile_patch_tmp, stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + LOGGER.debug('patch_first_file(return code): %r', out) + + for i in cursor: + id_dba = i["_id"] + # set prev id in current doc. + self.prev_id = id_dba + file_db = self.dbm.get_file_from_db_raw(id_dba) + patch_file = open(self.tempfile_patch_tmp, 'w') + patch_file.write(file_db) + patch_file.close() + + # # gen. tmp all version files + self.list_tmp_files.append((self.tempfilefd_ver, + self.tempfile_ver + + str(files_count))) + + if BlackListCompacter.generate_all_file: + # # generate all partial files + out = subprocess.call("patch " + self.tempfile_ver + + str(files_count - 1) + " -i " + + self.tempfile_patch_tmp + " -o " + + self.tempfile_ver + str(files_count), + stdout=f_sout, stderr=subprocess.STDOUT, shell=True) + os.chmod(self.tempfile_ver + str(files_count), 0644) + LOGGER.debug('patch_all_files(return code): %r', out) + + else: + out = subprocess.call( + "patch " + self.tempfile_file_recovery_0 + " -i " + self.tempfile_patch_tmp, + stdout=f_sout, stderr=subprocess.STDOUT, shell=True + ) + LOGGER.debug('patch(return code): %r', out) + + files_count += 1 + f_sout.close() + return self.tempfile_file_recovery_0 + + def start(self): + """Start BlackListCompacter.""" + LOGGER.debug('BlackListCompacter.GENERATE_ALL_FILE: %r', + BlackListCompacter.generate_all_file) + LOGGER.debug('BlackListCompacter.PERIOD: %r', BlackListCompacter.period) + LOGGER.debug('BlackListCompacter.INIT: %r', BlackListCompacter.init) + file_id = None + try: + file_id, cursor = self.get_patches() + LOGGER.debug('file_id:%r, cursor:%r:', file_id, cursor) + # count files orig + diffs + # cursor.count it is ok, if we count from zero(marker=0) + files_count = cursor.count() + + self.marker_db_diff = files_count + LOGGER.debug('files_count: %r', files_count) + except StopIteration, exc: + # first file + LOGGER.warning('First file, initialize: %r', exc) + BlackListCompacter.init = 1 + # init files, set marker = 0 + self.marker_db_diff = self.marker_db_init + # init file, set prev id = null + self.prev_id = None + + if file_id: # patch_start exist + if files_count <= BlackListCompacter.period: + # add new patch_diffs.txt in DB + BlackListCompacter.init = 0 + orig = self.generate_orig_file(cursor, file_id) + self.save_diff_in_db((orig, self.file_init)) + else: + # # generate new patch_start.txt, and save to DB + BlackListCompacter.init = 1 + self.save_diff_in_db((self.tempfile_file, self.file_init)) + else: + # failure to file patch_start.txt, initialize new cycle + BlackListCompacter.init = 1 + self.save_diff_in_db((self.tempfile_file, self.file_init)) + + self.cleanup() + + def cleanup(self): + """Cleanup all tmp files.""" + + for fd, fn in self.list_tmp_files: + if os.path.exists(fn): + os.remove(fn) + LOGGER.debug('cleanup tmp files') + + +def main(): + with logging_configured(): + t = DBarchiver() + try: + t.run() + except KeyboardInterrupt: + LOGGER.debug('SIGINT. waiting for ...') + t.stop() + except socket.timeout as exc: + # at the moment need to capture sys.exit tool for monitoring + LOGGER.critical('socket.timeout: %r', exc) + print >> sys.stderr, exc + sys.exit(1) + except socket.error as exc: + # at the moment need to capture sys.exit tool for monitoring + LOGGER.critical('socket.error: %r', exc) + print >> sys.stderr, exc + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/N6Core/n6/archiver/mysqldb_patch.py b/N6Core/n6/archiver/mysqldb_patch.py new file mode 100644 index 0000000..9e2100a --- /dev/null +++ b/N6Core/n6/archiver/mysqldb_patch.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2018 NASK. All rights reserved. +# +# Copyright (c) 2014 the author(s) of the MySQLdb1 library +# (GPL-licensed), see below... All rights reserved. + +""" +This module patches a few MySQLdb library functions -- to enhance +warnings (adding information about the query and its args). + +Import this module before the importing sqlalchemy. +""" + +### XXX: it would be nice to check if it is possible to patch smaller +### portions of the code to obtain the same effect... + +### FIXME: fix indentation in this module + + +import sys + +from MySQLdb import cursors + +from n6lib.log_helpers import get_logger +from n6lib.common_helpers import provide_surrogateescape + +provide_surrogateescape() +LOGGER = get_logger(__name__) + + +warning_standard = False +warning_details_to_logs = True + +insert_values = cursors.insert_values +ProgrammingError = cursors.ProgrammingError + + +# modified MySQLdb.cursors.BaseCursor.execute() (changes marked with `###`) +def execute(self, query, args=None): + + """Execute a query. + + query -- string, query to execute on server + args -- optional sequence or mapping, parameters to use with query. + + Note: If args is a sequence, then %s must be used as the + parameter placeholder in the query. If a mapping is used, + %(key)s must be used as the placeholder. + + Returns long integer rows affected, if any + + """ + del self.messages[:] + db = self._get_db() + if isinstance(query, unicode): + query = query.encode(db.unicode_literal.charset) + if args is not None: + if isinstance(args, dict): + query = query % dict((key, db.literal(item)) + for key, item in args.iteritems()) + else: + query = query % tuple([db.literal(item) for item in args]) + try: + r = None + r = self._query(query) + except TypeError, m: + if m.args[0] in ("not enough arguments for format string", + "not all arguments converted"): + self.messages.append((ProgrammingError, m.args[0])) + self.errorhandler(self, ProgrammingError, m.args[0]) + else: + self.messages.append((TypeError, m)) + self.errorhandler(self, TypeError, m) + except (SystemExit, KeyboardInterrupt): + raise + except: + exc, value, tb = sys.exc_info() + del tb + self.messages.append((exc, value)) + self.errorhandler(self, exc, value) + self._executed = query + ### orig: if not self._defer_warnings: self._warning_check() + if not self._defer_warnings: self._warning_check(query, args=args) + return r + + +# modified MySQLdb.cursors.BaseCursor.executemany() (changes marked with `###`) +def executemany(self, query, args): + + """Execute a multi-row query. + + query -- string, query to execute on server + + args + + Sequence of sequences or mappings, parameters to use with + query. + + Returns long integer rows affected, if any. + + This method improves performance on multiple-row INSERT and + REPLACE. Otherwise it is equivalent to looping over args with + execute(). + + """ + del self.messages[:] + db = self._get_db() + if not args: return + if isinstance(query, unicode): + query = query.encode(db.unicode_literal.charset) + m = insert_values.search(query) + if not m: + r = 0 + for a in args: + r = r + self.execute(query, a) + return r + p = m.start(1) + e = m.end(1) + qv = m.group(1) + try: + q = [] + for a in args: + if isinstance(a, dict): + q.append(qv % dict((key, db.literal(item)) + for key, item in a.iteritems())) + else: + q.append(qv % tuple([db.literal(item) for item in a])) + except TypeError, msg: + if msg.args[0] in ("not enough arguments for format string", + "not all arguments converted"): + self.errorhandler(self, ProgrammingError, msg.args[0]) + else: + self.errorhandler(self, TypeError, msg) + except (SystemExit, KeyboardInterrupt): + raise + except: + exc, value, tb = sys.exc_info() + del tb + self.errorhandler(self, exc, value) + r = self._query('\n'.join([query[:p], ',\n'.join(q), query[e:]])) + ### orig: if not self._defer_warnings: self._warning_check() + if not self._defer_warnings: self._warning_check(query, args=args) + return r + +# modified MySQLdb.cursors.BaseCursor._warning_check() (changes marked with `###`) +def _warning_check(self, query=None, args=None): + from warnings import warn + if self._warnings: + warnings = self._get_db().show_warnings() + if warnings: + # This is done in two loops in case + # Warnings are set to raise exceptions. + for w in warnings: + self.messages.append((self.Warning, w)) + for w in warnings: + ### orig: warn(w[-1], self.Warning, 3) + if warning_standard: + warn(w[-1], self.Warning, 3) + if warning_details_to_logs: + LOGGER.warning( + '%s, QUERY_SQL: %r, ARGS: %r', + w[-1].encode('utf-8'), query, args) + elif self._info: + self.messages.append((self.Warning, self._info)) + warn(self._info, self.Warning, 3) + + +cursors.BaseCursor.execute = execute +cursors.BaseCursor.executemany = executemany +cursors.BaseCursor._warning_check = _warning_check diff --git a/N6Core/n6/archiver/recorder.py b/N6Core/n6/archiver/recorder.py new file mode 100644 index 0000000..86bc7c0 --- /dev/null +++ b/N6Core/n6/archiver/recorder.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2014 NASK. All rights reserved. + +""" +The *recorder* component -- adds n6 events to the database. +""" + +### TODO: this module is to be replaced with a new implementation... + +import datetime +import logging +import os +import sys + +import n6.archiver.mysqldb_patch + +from sqlalchemy import create_engine +from sqlalchemy.exc import IntegrityError, OperationalError + +from n6.base.queue import QueuedBase +from n6lib.config import Config +from n6lib.data_backend_api import N6DataBackendAPI +from n6lib.datetime_helpers import parse_iso_datetime_to_utc +from n6lib.db_events import n6ClientToEvent, n6NormalizedData +from n6lib.log_helpers import get_logger, logging_configured +from n6lib.record_dict import RecordDict, BLRecordDict +from n6lib.transaction_helpers import transact +from n6lib.common_helpers import replace_segment + + +### MySQLdb warnings monkey-patching: +### to write to stderr only, set: +# n6.archiver.mysqldb_patch.warning_standard = True +# n6.archiver.mysqldb_patch.warning_details_to_logs = False +### to write to logs only, set: +# n6.archiver.mysqldb_patch.warning_standard = False +# n6.archiver.mysqldb_patch.warning_details_to_logs = True + +logging.basicConfig() + +## many logs from db: +#logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG) + +LOGGER = get_logger(__name__) + + +class PublishError(Exception): + """Exeption used by SourceTransfer class""" + + +class Recorder(QueuedBase): + """Save record in zbd queue.""" + input_queue = {"exchange": "event", + "exchange_type": "topic", + "queue_name": 'zbd', + "binding_keys": ['event.filtered.*.*', + 'bl-new.filtered.*.*', + 'bl-change.filtered.*.*', + 'bl-delist.filtered.*.*', + 'bl-expire.filtered.*.*', + 'bl-update.filtered.*.*', + 'suppressed.filtered.*.*', + ] + } + + output_queue = {"exchange": "event", + "exchange_type": "topic" + } + + SQL_WAIT_TIMEOUT = "SET SESSION wait_timeout = {wait}" + + def __init__(self, **kwargs): + LOGGER.info("Recorder Start") + config = Config(required={"recorder": ("uri", "echo")}) + self.config = config["recorder"] + self.rows = None + self.record_dict = None + self.source = None + self.dir_name = None + self.wait_timeout = int(self.config.get("wait_timeout", 28800)) + engine = create_engine(self.config["uri"], echo=bool((int(self.config["echo"])))) + self.session_db = N6DataBackendAPI.configure_db_session(engine) + self.set_session_wait_timeout() + self.records = None + self.routing_key = None + + self.dict_map_fun = { + "event.filtered": (RecordDict.from_json, self.new_event), + "bl-new.filtered": (BLRecordDict.from_json, self.blacklist_new), + "bl-change.filtered": (BLRecordDict.from_json, self.blacklist_change), + "bl-delist.filtered": (BLRecordDict.from_json, self.blacklist_delist), + "bl-expire.filtered": (BLRecordDict.from_json, self.blacklist_expire), + "bl-update.filtered": (BLRecordDict.from_json, self.blacklist_update), + "suppressed.filtered": (RecordDict.from_json, self.suppressed_update), + } + # keys in each of the tuples being values of `dict_map_fun` + self.FROM_JSON = 0 + self.HANDLE_EVENT = 1 + + super(Recorder, self).__init__(**kwargs) + + def ping_connection(self): + """ + Required to maintain the connection to MySQL. + Perform ping before each query to the database. + OperationalError if an exception occurs, remove sessions, and connects again. + Set the wait_timeout(Mysql session variable) for the session on self.wait_timeout. + """ + try: + self.session_db.execute("SELECT 1") + except OperationalError as exc: + # OperationalError: (2006, 'MySQL server has gone away') + LOGGER.warning("Database server went away: %r", exc) + LOGGER.info("Reconnect to server") + self.session_db.remove() + self.set_session_wait_timeout() + + def set_session_wait_timeout(self): + """set session wait_timeout in mysql SESSION VARIABLES""" + self.session_db.execute(Recorder.SQL_WAIT_TIMEOUT.format(wait=self.wait_timeout)) + + @staticmethod + def get_truncated_rk(rk, parts): + """ + Get only a part of the given routing key. + + Args: + `rk`: routing key. + `parts`: number of dot-separated parts (segments) to be kept. + + Returns: + Truncated `rk` (containing only first `parts` segments). + + >>> Recorder.get_truncated_rk('111.222.333.444', 0) + '' + >>> Recorder.get_truncated_rk('111.222.333.444', 1) + '111' + >>> Recorder.get_truncated_rk('111.222.333.444', 2) + '111.222' + >>> Recorder.get_truncated_rk('111.222.333.444', 3) + '111.222.333' + >>> Recorder.get_truncated_rk('111.222.333.444', 4) + '111.222.333.444' + >>> Recorder.get_truncated_rk('111.222.333.444', 5) # with log warning + '111.222.333.444' + """ + rk = rk.split('.') + parts_rk = [] + try: + for i in xrange(parts): + parts_rk.append(rk[i]) + except IndexError: + LOGGER.warning("routing key %r contains less than %r segments", rk, parts) + return '.'.join(parts_rk) + + def input_callback(self, routing_key, body, properties): + """ Channel callback method """ + # first let's try ping mysql server + self.ping_connection() + self.records = {'event': [], 'client': []} + self.routing_key = routing_key + + # take the first two parts of the routing key + truncated_rk = self.get_truncated_rk(self.routing_key, 2) + + # run BLRecordDict.from_json() or RecordDict.from_json() + # depending on the routing key + from_json = self.dict_map_fun[truncated_rk][self.FROM_JSON] + self.record_dict = from_json(body) + # add modified time, set microseconds to 0, because the database + # does not have microseconds, and it is not known if the base is not rounded + self.record_dict['modified'] = datetime.datetime.utcnow().replace(microsecond=0) + # run the handler method corresponding to the routing key + handle_event = self.dict_map_fun[truncated_rk][self.HANDLE_EVENT] + with self.setting_error_event_info(self.record_dict): + handle_event() + + assert 'source' in self.record_dict + LOGGER.debug("source: %r", self.record_dict['source']) + LOGGER.debug("properties: %r", properties) + #LOGGER.debug("body: %r", body) + + def json_to_record(self, rows): + """ + Deserialize json to record db.append. + + Args: `rows`: row from RecordDict + """ + if 'client' in rows[0]: + for client in rows[0]['client']: + tmp_rows = rows[0].copy() + tmp_rows['client'] = client + self.records['client'].append(tmp_rows) + + def insert_new_event(self, items, with_transact=True, recorded=False): + """ + New events and new blacklist add to database, + default in the transaction, or the outer transaction(with_transact=False). + """ + try: + if with_transact: + with transact: + self.session_db.add_all(items) + else: + self.session_db.add_all(items) + except IntegrityError as exc: + LOGGER.warning("IntegrityError %r", exc) + else: + if recorded and not self.cmdline_args.n6recovery: + rk = replace_segment(self.routing_key, 1, 'recorded') + LOGGER.debug( + 'Publish for email notifications ' + '-- rk: %r, record_dict: %r', + rk, self.record_dict) + self.publish_event(self.record_dict, rk) + + def publish_event(self, data, rk): + """ + Publishes event to the output queue. + + Args: + `data`: data from recorddict + `rk` : routing key + """ + body = data.get_ready_json() + self.publish_output(routing_key=rk, body=body) + + def new_event(self, _is_blacklist=False): + """ + Add new event to n6 database. + """ + LOGGER.debug('* new_event() %r', self.record_dict) + + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + if _is_blacklist: + event_record["status"] = "active" + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + items = [] + for record in self.records['event']: + event = n6NormalizedData(**record) + items.append(event) + + for record in self.records['client']: + client = n6ClientToEvent(**record) + items.append(client) + + LOGGER.debug("insert new events, count.: %r", len(items)) + self.insert_new_event(items, recorded=True) + + def blacklist_new(self): + self.new_event(_is_blacklist=True) + + def blacklist_change(self): + """ + Black list change(change status to replaced in existing blacklist event, + and add new event in changing values(new id, and old replaces give comparator)). + """ + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + id_db = self.records['event'][0]["id"] + id_replaces = self.records['event'][0]["replaces"] + LOGGER.debug("ID: %r REPLACES: %r", id_db, id_replaces) + + try: + with transact: + rec_count = (self.session_db.query(n6NormalizedData). + filter(n6NormalizedData.id == id_replaces). + update({'status': 'replaced', + 'modified': datetime.datetime.utcnow().replace(microsecond=0) + })) + + with transact: + items = [] + for record in self.records['event']: + record["status"] = "active" + event = n6NormalizedData(**record) + items.append(event) + + for record in self.records['client']: + client = n6ClientToEvent(**record) + items.append(client) + + if rec_count: + LOGGER.debug("insert new events, count.: %r", len(items)) + else: + LOGGER.debug("bl-change, records with id %r DO NOT EXIST!", id_replaces) + LOGGER.debug("inserting new events anyway, count.: %r", len(items)) + self.insert_new_event(items, with_transact=False, recorded=True) + + except IntegrityError as exc: + LOGGER.warning("IntegrityError: %r", exc) + + def blacklist_delist(self): + """ + Black list delist (change status to delisted in existing blacklist event). + """ + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + id_db = self.records['event'][0]["id"] + LOGGER.debug("ID: %r STATUS: %r", id_db, 'delisted') + + with transact: + (self.session_db.query(n6NormalizedData). + filter(n6NormalizedData.id == id_db). + update( + { + 'status': 'delisted', + 'modified': datetime.datetime.utcnow().replace(microsecond=0), + })) + + def blacklist_expire(self): + """ + Black list expire (change status to expired in existing blacklist event). + """ + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + + id_db = self.records['event'][0]["id"] + LOGGER.debug("ID: %r STATUS: %r", id_db, 'expired') + + with transact: + (self.session_db.query(n6NormalizedData). + filter(n6NormalizedData.id == id_db). + update( + { + 'status': 'expired', + 'modified': datetime.datetime.utcnow().replace(microsecond=0), + })) + + def blacklist_update(self): + """ + Black list update (change expires to new value in existing blacklist event). + """ + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + id_event = self.records['event'][0]["id"] + expires = self.records['event'][0]["expires"] + LOGGER.debug("ID: %r NEW_EXPIRES: %r", id_event, expires) + + with transact: + rec_count = (self.session_db.query(n6NormalizedData). + filter(n6NormalizedData.id == id_event). + update({'expires': expires, + 'modified': datetime.datetime.utcnow().replace(microsecond=0), + })) + if rec_count: + LOGGER.debug("records with the same id %r exist: %r", + id_event, rec_count) + else: + items = [] + for record in self.records['event']: + record["status"] = "active" + event = n6NormalizedData(**record) + items.append(event) + + for record in self.records['client']: + client = n6ClientToEvent(**record) + items.append(client) + LOGGER.debug("bl-update, records with id %r DO NOT EXIST!", id_event) + LOGGER.debug("insert new events,::count:: %r", len(items)) + self.insert_new_event(items, with_transact=False) + + def suppressed_update(self): + """ + Agregated event update(change fields: until and count, to the value of suppressed event). + """ + LOGGER.debug('* suppressed_update() %r', self.record_dict) + + # add event records from RecordDict + for event_record in self.record_dict.iter_db_items(): + self.records['event'].append(event_record) + + self.json_to_record(self.records['event']) + id_event = self.records['event'][0]["id"] + until = self.records['event'][0]["until"] + count = self.records['event'][0]["count"] + + # optimization: we can limit time => searching within one partition, not all; + # it seems that mysql (and/or sqlalchemy?) truncates times to seconds, + # we are also not 100% sure if other time data micro-distortions are not done + # -- that's why here we use a 1-second-range instead of an exact value + first_time_min = parse_iso_datetime_to_utc( + self.record_dict["_first_time"]).replace(microsecond=0) + first_time_max = first_time_min + datetime.timedelta(days=0, seconds=1) + + with transact: + rec_count = (self.session_db.query(n6NormalizedData) + .filter( + n6NormalizedData.time >= first_time_min, + n6NormalizedData.time <= first_time_max, + n6NormalizedData.id == id_event) + .update({'until': until, 'count': count})) + if rec_count: + LOGGER.debug("records with the same id %r exist: %r", + id_event, rec_count) + else: + items = [] + for record in self.records['event']: + event = n6NormalizedData(**record) + items.append(event) + + for record in self.records['client']: + client = n6ClientToEvent(**record) + items.append(client) + LOGGER.warning("suppressed_update, records with id %r DO NOT EXIST!", id_event) + LOGGER.debug("insert new events,,::count:: %r", len(items)) + self.insert_new_event(items, with_transact=False) + + +def main(): + with logging_configured(): + if 'n6integration_test' in os.environ: + # for debugging only + LOGGER.setLevel(logging.DEBUG) + LOGGER.addHandler(logging.StreamHandler(stream=sys.__stdout__)) + d = Recorder() + try: + d.run() + except KeyboardInterrupt: + d.stop() + + +if __name__ == "__main__": + main() diff --git a/N6Core/n6/base/__init__.py b/N6Core/n6/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/N6Core/n6/base/config.py b/N6Core/n6/base/config.py new file mode 100644 index 0000000..d394878 --- /dev/null +++ b/N6Core/n6/base/config.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2018 NASK. All rights reserved. + +import os +import os.path +import shutil +import sys + +from n6lib.const import USER_DIR, ETC_DIR + + +# NOTE: more of the config-related stuff is in n6lib.config + + +def install_default_config(): + """ + Copy default N6Core conf files to '/etc/n6' or '~/.n6'. + """ + + def confirm_yes_no(msg, default=True): + print "%s [%s]" % (msg, "Y/n" if default else "y/N") + # raw_input returns the empty string for "enter" + yes = set(['yes', 'y', 'ye']) + no = set(['no', 'n']) + if default: + yes.add("") + else: + no.add("") + while True: + choice = raw_input().lower() + if choice in yes: + return True + elif choice in no: + return False + else: + sys.stdout.write("Please respond with 'yes' or 'no': ") + + def check_existing_dir_content(install_to, alternative_to): + if os.path.isdir(install_to) and os.listdir(install_to): + if confirm_yes_no( + "Directory '%s' is not empty. Remove existing files?" % install_to, + default=False): + shutil.rmtree(install_to) + os.mkdir(install_to) + elif alternative_to is not None and confirm_yes_no( + "Install in '%s' instead?" % alternative_to): + install_to = alternative_to + alternative_to = None + if os.path.isdir(install_to) and os.listdir(install_to): + if confirm_yes_no( + "Directory '%s' is not empty. Remove existing files?" % install_to, + default=False): + shutil.rmtree(install_to) + os.mkdir(install_to) + else: + print "Ok. Exiting" + sys.exit(0) + else: + print "Ok. Exiting" + sys.exit(0) + return install_to, alternative_to + + if not confirm_yes_no("Copy sample configuration files to the system?"): + print "Ok. Exiting" + sys.exit(0) + + etcdir = ETC_DIR + userdir = USER_DIR + + if not os.path.isdir(etcdir): + try: + os.makedirs(etcdir) + except (OSError, IOError): + pass + + if os.access(etcdir, os.W_OK): + install_to = etcdir + alternative_to = userdir + elif confirm_yes_no("No write access to '%s'. Write to '%s' instead?" % (etcdir, userdir)): + install_to = userdir + alternative_to = None + else: + print "Ok. Exiting" + sys.exit(0) + + install_to, alternative_to = check_existing_dir_content(install_to, alternative_to) + + + from pkg_resources import ( + Requirement, + resource_filename, + resource_listdir, + cleanup_resources) #@UnresolvedImport + + try: + config_template_dir = 'n6/data/conf/' + files = resource_listdir(Requirement.parse("n6"), config_template_dir) + for f in files: + filename = resource_filename(Requirement.parse("n6"), os.path.join(config_template_dir, f)) + try: + if not os.path.isdir(install_to): + os.makedirs(install_to) + shutil.copy(filename, os.path.join(install_to, f)) + except (IOError, OSError): + if alternative_to is not None and confirm_yes_no( + "Cannot create config files in '%s'. " + "Create in '%s' instead?" % (install_to, alternative_to)): + install_to, _ = check_existing_dir_content(alternative_to, None) + try: + if not os.path.isdir(install_to): + os.makedirs(install_to) + shutil.copy(filename, os.path.join(install_to, f)) + except (IOError, OSError): + print "Error while copying sample conf files to '%s'. Exiting." % install_to + sys.exit(1) + else: + print "Ok. Exiting" + sys.exit(0) + finally: + cleanup_resources() + print "Success." diff --git a/N6Core/n6/base/queue.py b/N6Core/n6/base/queue.py new file mode 100644 index 0000000..307844b --- /dev/null +++ b/N6Core/n6/base/queue.py @@ -0,0 +1,1050 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2018 NASK. All rights reserved. + +# Note, however, that some parts of the QueuedBase class are patterned +# after some examples from the docs of a 3rd-party library: `pika`; and +# some of the docstrings are taken from or contain fragments of the +# docs of the `pika` library. + +import functools +import sys +import types + +try: + import argparse +except ImportError: + print >>sys.stderr, "Warning: argparse is required to run AMQP components" +import contextlib +import copy +import pprint +import re + +try: + import pika + import pika.credentials +except ImportError: + print >>sys.stderr, "Warning: pika is required to run AMQP components" + + +from n6lib.amqp_helpers import get_amqp_connection_params_dict +from n6lib.argument_parser import N6ArgumentParser +from n6lib.auth_api import AuthAPICommunicationError +from n6lib.log_helpers import get_logger + + +LOGGER = get_logger(__name__) + + +class n6QueueProcessingException(Exception): + pass + + +class n6AMQPCommunicationError(Exception): + pass + + +class QueuedBase(object): + + """ + Base class for n6 components that communicate through AMQP queues. + + Child classes should define the following class attributes: + + * `input_queue` and/or + * `output_queue` + + -- to get respective queues configured. + + Each of them should either be None or an appropriate collection: + + * input_queue should be a dict: + { + "exchange": "", + "exchange_type": "", + "queue_name": "", + "binding_keys": , + "queue_exclusive": True|False, # is queue exclusive (optional) + } + + * output_queue should be a dict or a list of such dicts -- each like: + { + "exchange": "", + "exchange_type": "", + } + + These two attributes are automatically deep-copied before being + transformed into instance-specific attributes (then adjusted + per-instance, see: __new__() and preinit_hook()...). + + QueuedBase should handle unexpected interactions with RabbitMQ + such as channel and connection closures. + + Dev note: if child classes are defining __init__(), it should accept + **kwargs and call super(ChildClass, self).__init__(**kwargs) + """ + + # + # Basic attributes + + CONNECTION_ATTEMPTS = 600 + CONNECTION_RETRY_DELAY = 10 + SOCKET_TIMEOUT = 3.0 + + # the name of the config section the RabbitMQ settings shall be taken from + rabbitmq_config_section = 'rabbitmq' + + # (see: the __new__() class method below) + input_queue = None + output_queue = None + + # if a script should run only in one instance - used to set basic_consume(exclusive=) flag + single_instance = True + + # in a subclass, it should be set to False if the component should not + # accept --n6recovery argument option (see: the get_arg_parser() method) + supports_n6recovery = True + + # it is set on a new instance by __new__() (which is called + # automatically before __init__()) to an argparse.Namespace instance + cmdline_args = None + + # parameter prefetch_count + # Specifies a prefetch window in terms of whole messages. + # This field may be used in combination with the prefetch-size field + # (although the prefetch-size limit is not implemented + # yet in RabbitMQ). A message will only be sent in advance + # if both prefetch windows (and those at the channel + # and connection level) allow it. The prefetch-count is ignored + # if the no-ack option is set. + prefetch_count = 20 + + # basic kwargs for pika.BasicProperties (message-publishing-related) + basic_prop_kwargs = {'delivery_mode': 2} + + # (in seconds) + reconnect_delay = 5 + + + # + # Pre-init methods + + # (for historical reasons we do not want to place these operations + # in __init__() -- mainly because __init__() is skipped in several + # unit tests...) + + def __new__(cls, **kwargs): + """ + Create and pre-configure an instance. + + Normally, this special method should not be overridden in + subclasses. (If you really need that please *extend* it by + overridding and calling with super()). + + The method causes that immediately after creating of a + QueuedBase-derived class instance -- before calling __init__() + -- the following operations are performed on the instance: + + 1) (re)-making the `input_queue` and `output_queue` attributes + as instance ones -- by deep-copying them (so they are + exclusively owned by the instance and not by the class or any + of its superclasses); note: whereas `input_queue` (both as + the class attribute and the resultant instance attribute) + should always be a dict (unless None), the `output_queue` + *instance* attribute must always be a list of dicts (unless + None) -- i.e.: if the `output_queue` *class* attribute is a + dict the resultant *instance* attribute will be a one-element + list (containing a deep copy of that dict); + + 2) the get_arg_parser() method is called to obtain the argument + parser object; + + 3) the parse_known_args() method of the obtained argument parser + is called and the obtained command-line argument container + (produced by the argument parser as a argparse.Namespace + instance) is set as the `cmdline_args` attribute. + + 4) the preinit_hook() method is called (see its docs...). + """ + # some unit tests are over-zealous about patching super() + from __builtin__ import super + + self = super(QueuedBase, cls).__new__(cls, **kwargs) + + if cls.input_queue is not None and not isinstance(self.input_queue, dict): + raise TypeError('The `input_queue` class attribute must be a dict or None') + self.input_queue = copy.deepcopy(cls.input_queue) + + if cls.output_queue is not None and not ( + isinstance(self.output_queue, dict) or + isinstance(self.output_queue, list) and all( + isinstance(item, dict) for item in self.output_queue)): + raise TypeError('The `output_queue` class attribute must be ' + 'a dict or a list of dicts, or None') + output_queue = copy.deepcopy(cls.output_queue) + if isinstance(output_queue, dict): + output_queue = [output_queue] + self.output_queue = output_queue + + self.cmdline_args = self.parse_cmdline_args() + self.preinit_hook() + return self + + def parse_cmdline_args(self): + """ + Parse commandline arguments (taken from sys.argv[1:]). + + Returns: + An argparse.Namespace instance containing parsed commandline + arguments. + + Unrecognized commandline arguments starting with the 'n6' text + prefixed by one or more '-' (hyphen) characters (such as + '-n6recovery' or '--n6blahblah'...) cause the SystemExit + exception with exit code 2; other unrecognized commandline + arguments are ignored. + + This method is automatically called after instance creation, + before __init__() is called (see the docs of __new__()). + + This method *should not* be overridden completely; instead, it + can be *extended* (overridden + called with super()). + """ + arg_parser = self.get_arg_parser() + cmdline_args, unknown = arg_parser.parse_known_args() + illegal_n6_args = [arg for arg in unknown if re.match(r'\-+n6', arg)] + if illegal_n6_args: + arg_parser.error('unrecognized n6-specific arguments: {0}'.format( + ', '.join(illegal_n6_args))) + return cmdline_args + + def get_arg_parser(self): + """ + Make and configure argument parser. + + Returns: + An argparse.ArgumentParser instance. + + This method is automatically called after instance creation, + before __init__() is called (see the docs of __new__() and + parse_cmdline_args()). + + This method *should not* be overridden completely; instead, it + can be *extended* (overridden + called with super()). + + The default implementation of this method adds to the created + argument parser the possibility to run (from the command line) + parsers/collectors/other components that inherit from the + QueuedBase class -- with the "--n6recovery" parameter; it will + cause that the standard implementation of the preinit_hook() + method will add the '_recovery' suffix to all AMQP exchange + and queue names. + + To prevent this method from providing the "--n6recovery" + parameter, set the `supports_n6recovery` class attribute to + False. + """ + arg_parser = N6ArgumentParser() + if self.supports_n6recovery: # <- True by default + arg_parser.add_argument('--n6recovery', + action='store_true', + help=('add the "_recovery" suffix to ' + 'all AMQP exchange/queue names')) + return arg_parser + + def preinit_hook(self): + """ + Adjust some attributes after instance creation, before __init__() call. + + This method is automatically called after instance creation, + before __init__() is called (see: the docs of __new__()). + + This method *should not* be overridden completely; instead, + it can be *extended* (overridden + called with super()). + + The default implementation of this method checks whether + self.cmdline_args.n6recovery is set to true; if it is then the + '_recovery' suffix is added to AMQP exchange and queue names + in the `input_queue` and `output_queue` instance attributes + (it is needed to perform data recovery from MongoDB...). + """ + if not self.supports_n6recovery or not self.cmdline_args.n6recovery: + return + suffix = '_recovery' + assert ('input_queue' in vars(self) and # __new__() ensures + 'output_queue' in vars(self)) # that this is true + queue_conf_dicts = [] + if self.input_queue is not None: + assert isinstance(self.input_queue, dict) # it's dict + queue_conf_dicts.append(self.input_queue) # so using .append + if self.output_queue is not None: + assert isinstance(self.output_queue, list) # it's list of dicts + queue_conf_dicts.extend(self.output_queue) # so using .extend + for conf_dict in queue_conf_dicts: + for key in ('exchange', 'queue_name'): + if key in conf_dict: + conf_dict[key] += suffix + + + # + # Actual initialization + + def __init__(self, **kwargs): + super(QueuedBase, self).__init__(**kwargs) + + LOGGER.debug('input_queue: %r', self.input_queue) + LOGGER.debug('output_queue: %r', self.output_queue) + + self._connection = None + self._channel_in = None + self._channel_out = None + self._num_queues_bound = 0 + self._declared_output_exchanges = set() + self.output_ready = False + self.waiting_for_reconnect = False + self._closing = False + self._consumer_tag = None + self._conn_params_dict = self.get_connection_params_dict() + + + # + # Utility static methods + + @classmethod + def get_connection_params_dict(cls): + """ + Get the AMQP connection parameters (as a dict) + using n6lib.amqp_helpers.get_amqp_connection_params_dict() + and the `CONNECTION_ATTEMPTS`, `CONNECTION_RETRY_DELAY` and + `SOCKET_TIMEOUT` class constants. + + Returns: + A dict that can be used as **kwargs for pika.ConnectionParameters. + """ + conn_params_dict = get_amqp_connection_params_dict(cls.rabbitmq_config_section) + conn_params_dict.update( + connection_attempts=cls.CONNECTION_ATTEMPTS, + retry_delay=cls.CONNECTION_RETRY_DELAY, + socket_timeout=cls.SOCKET_TIMEOUT, + ) + return conn_params_dict + + + # + # Regular instance methods + + # Start/stop-related stuff: + + def run(self): + """Connecting to RabbitMQ and start the IOLoop (blocking on it).""" + self._connection = self.connect() + self._connection.ioloop.start() + + ### XXX... (TODO: analyze whether it is correct...) + def stop(self): + """ + Quote from pika docs: + + ''' + Cleanly shutdown the connection to RabbitMQ by stopping the consumer + with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok + will be invoked by pika, which will then closing the channel and + connection. The IOLoop is started again because this method is invoked + when CTRL-C is pressed raising a KeyboardInterrupt exception. This + exception stops the IOLoop which needs to be running for pika to + communicate with RabbitMQ. All of the commands issued prior to starting + the IOLoop will be buffered but not processed. + ''' + """ + LOGGER.debug('Stopping') + self.inner_stop() + self._connection.ioloop.start() + LOGGER.info('Stopped') + + ### XXX... (TODO: analyze whether it is correct...) + def inner_stop(self): + self._closing = True + self.stop_consuming() + self.close_channels() + + # Connection-related stuff: + + def connect(self): + """ + From pika docs: + + This method connects to RabbitMQ, returning the connection handle. + When the connection is established, the on_connection_open method + will be invoked by pika. + + Returns: + pika.SelectConnection + """ + LOGGER.info('Connecting to %s', self._conn_params_dict['host']) + + return pika.SelectConnection( + pika.ConnectionParameters(**self._conn_params_dict), + self.on_connection_open, + self.on_connection_error_open, + stop_ioloop_on_close=False, + ) + + def close_connection(self): + LOGGER.info('Closing connection...') + self._connection.close() + + def on_connection_error_open(self, connection): + LOGGER.critical('Could not connect to RabbitMQ after %d attempts', + connection.params.connection_attempts) + sys.exit(1) + + def on_connection_open(self, connection): + """ + From pika docs: + + This method is called by pika once the connection to RabbitMQ has + been established. It passes the handle to the connection object. + + Args: + `connection`: pika.SelectConnection instance + """ + LOGGER.info('Connection opened') + self._connection.add_on_close_callback(self.on_connection_closed) + self.open_channels() + + # WARNING: probably due to some bug in some libraries, this callback + # may be called more than once per one connection breakage -- so this + # callback should be idempotent (that's why the `waiting_for_reconnect` + # flag has been introduced) + def on_connection_closed(self, connection, reply_code, reply_text): + """ + From pika docs: + + This method is invoked by pika when the connection to RabbitMQ is + closed unexpectedly. Since it is unexpected, we will reconnect to + RabbitMQ if it disconnects. + + Args: + `connection`: The closed connection obj + `reply_code`: The server-provided reply_code if given + `reply_text`: The server-provided reply_text if given + """ + self._channel_in = None + self._channel_out = None + self.output_ready = False + if self._closing: + self._connection.ioloop.stop() + else: + if self.waiting_for_reconnect: + # it may happen as, probably due to some bug, pika may + # call the on_connection_closed() callback twice + # (see: some comments in the ticket #2566) + LOGGER.warning( + 'Connection closed (not scheduling reopening as ' + 'it has already been scheduled!): (%s) %s', + reply_code, reply_text) + else: + LOGGER.warning( + 'Connection closed (reopening in %s seconds): (%s) %s', + self.reconnect_delay, reply_code, reply_text) + self.waiting_for_reconnect = True + self._connection.add_timeout(self.reconnect_delay, self.reconnect) + + def reconnect(self): + """ + From pika docs: + + Will be invoked by the IOLoop timer if the connection is + closed. See the on_connection_closed method. + """ + self.waiting_for_reconnect = False + self._connection.ioloop.stop() + if not self._closing: + self._connection = self.connect() + self._connection.ioloop.start() + + # Channel-related stuff: + + def open_channels(self): + """ + From pika docs (about .channel(): + + Open a new channel with RabbitMQ by issuing the Channel.Open RPC + command. When RabbitMQ responds that the channel is open, the + on_channel_open callback will be invoked by pika. + """ + LOGGER.info('Creating new channels') + if self.input_queue is not None: + self._connection.channel(on_open_callback=self.on_input_channel_open) + if self.output_queue is not None: + self._connection.channel(on_open_callback=self.on_output_channel_open) + + def close_channels(self): + """ + From pika docs (about .close()): + + Call to close [...] by issuing the Channel.Close RPC command. + """ + LOGGER.info('Closing the channels') + if self._channel_in is not None: + if self._channel_in.is_open: + self._channel_in.close() + if self._channel_out is not None: + if self._channel_out.is_open: + self._channel_out.close() + + def close_channel(self, channel_mode): + """ + Close the channel as specified by channel_mode. + + Args: + `channel_mode`: type of channel to close - "in" or "out" + """ + channel = getattr(self, "_channel_%s" % channel_mode) + if channel.is_open: + channel.close() + + def on_input_channel_open(self, channel): + """ + Invoked by pika when the input channel has been opened. + + Args: + `channel`: The channel object + """ + LOGGER.debug('Input channel opened') + self._channel_in = channel + self._channel_in.add_on_close_callback(self.on_channel_closed) + self._num_queues_bound = 0 + self.setup_input_exchange() + self.setup_dead_exchange() + + def on_output_channel_open(self, channel): + """ + Invoked by pika when the output channel has been opened. + + Args: + `channel`: The channel object + """ + LOGGER.debug('Output channel opened') + self._channel_out = channel + self._channel_out.add_on_close_callback(self.on_channel_closed) + self._declared_output_exchanges.clear() + self.setup_output_exchanges() + + def on_channel_closed(self, channel, reply_code, reply_text): + """ + From pika docs: + + Invoked by pika when a channel has been closed, e.g. when + RabbitMQ unexpectedly closes the channel. Channels can be closed + e.g. if you attempt to do something that violates the protocol, + such as re-declare an exchange or queue with different parameters. + In this case, we'll close the connection to shutdown the object. + + Args: + `channel`: The closed channel + `reply_code`: The numeric reason the channel was closed + `reply_text`: The text reason the channel was closed + """ + log = (LOGGER.debug if reply_code in (0, 200) + else LOGGER.warning) + log('Channel %i has been closed: (%s) %s', + channel, reply_code, reply_text) + self._connection.close( + reply_code=reply_code, + reply_text='Because channel {0} has been closed: "{1}"' + .format(channel, reply_text)) + + # Input-exchange/queue-related stuff: + + def setup_dead_exchange(self): + """Setup exchange for dead letter messages (e.g., rejected messages).""" + LOGGER.debug('Declaring dead-letter exchange') + if self._channel_in is not None: + self._channel_in.exchange_declare( + self.on_dead_exchange_declared, + "dead", + "topic", + durable=True) + else: + LOGGER.error('Dead-letter exchange cannot be declared because input channel is None') + ## XXX: restart or what? + + def on_dead_exchange_declared(self, frame): + """ + Called when dead letter exchange is created. + + Creates a queue to gather dead letter messages. + """ + LOGGER.debug('Declaring dead-letter queue') + if self._channel_in is not None: + assert frame.channel_number == self._channel_in.channel_number + self._channel_in.queue_declare( + self.on_dead_queue_declared, + "dead_queue", + durable=True, + auto_delete=False) + else: + LOGGER.error('Dead-letter queue cannot be declared because input channel is None') + ## XXX: restart or what? + + def on_dead_queue_declared(self, method_frame): + """ + Called when dead letter queue is created. + + Binds all keys to the queue to gather dead letter messages. + """ + LOGGER.debug('Binding dead-letter exchange') + if self._channel_in is not None: + self._channel_in.queue_bind( + self.on_bindok, + "dead_queue", + "dead", + "#") + else: + LOGGER.error('Dead-letter queue cannot be bound because input channel is None') + ## XXX: restart or what? + + def setup_input_exchange(self): + """ + From pika docs: + + Setup the [input] exchange on RabbitMQ by invoking the + Exchange.Declare RPC command. When it is complete, the + on_input_exchange_declared method will be invoked by pika. + """ + params = self.input_queue + LOGGER.debug('Declaring exchange %s', params["exchange"]) + self._channel_in.exchange_declare( + self.on_input_exchange_declared, + params["exchange"], + params["exchange_type"], + durable=True) + + def on_input_exchange_declared(self, frame): + """ + From pika docs: + + Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC + command. + + Args: + `frame`: Exchange.DeclareOk response frame + """ + LOGGER.debug('The input exchange declared') + if self._channel_in is not None: + assert frame.channel_number == self._channel_in.channel_number + self.setup_queue(self.input_queue["queue_name"]) + else: + LOGGER.error('Input queue cannot be set up because input channel is None') + ## XXX: restart or what? + + def setup_queue(self, queue_name): + """ + From pika docs: + + Setup the queue on RabbitMQ by invoking the Queue.Declare RPC + command. When it is complete, the on_queue_declared method will + be invoked by pika. + + Args: + `queue_name`: The name of the queue to declare. + """ + LOGGER.debug('Declaring queue %s', queue_name) + self._channel_in.queue_declare( + self.on_queue_declared, + queue_name, + durable=True, + auto_delete=False, + arguments={"x-dead-letter-exchange": "dead"}) + + def on_queue_declared(self, method_frame): + """ + From pika docs: + + Method invoked by pika when the Queue.Declare RPC call made in + setup_queue has completed. In this method we will bind the queue + and exchange together with the routing key by issuing the + Queue.Bind RPC command. When this command is complete, the + on_bindok method will be invoked by pika. + + Args: + method_frame: The Queue.DeclareOk frame + """ + LOGGER.debug('Binding %r to %r with %r', + self.input_queue["exchange"], + self.input_queue["queue_name"], + self.input_queue["binding_keys"]) + for binding_key in self.input_queue["binding_keys"]: + self._channel_in.queue_bind(self.on_bindok, + self.input_queue["queue_name"], + self.input_queue["exchange"], + binding_key) + + def on_bindok(self, unused_frame): + """ + From pika docs: + + Invoked by pika when the Queue.Bind method has completed. At this + point we will check if all needed bindings were created and + start consuming after that. + + Args: + unused_frame: The Queue.BindOk response frame + """ + LOGGER.debug('Queue bound') + self._num_queues_bound += 1 + # note: the dead-letter queue is also bound -- that's why we have `+ 1` below: + if self._num_queues_bound == len(self.input_queue["binding_keys"]) + 1: + LOGGER.debug('All queues bound (including the dead-letter queue)') + LOGGER.debug('Setting prefetch count') + self._channel_in.basic_qos(prefetch_count=self.prefetch_count) + self.start_consuming() + + def start_consuming(self): + """ + From pika docs: + + This method sets up the consumer by first calling + add_on_cancel_callback so that the object is notified if + RabbitMQ cancels the consumer. It then issues the Basic.Consume + RPC command which returns the consumer tag that is used to + uniquely identify the consumer with RabbitMQ. We keep the value + to use it when we want to cancel consuming. The on_message + method is passed in as a callback pika will invoke when a + message is fully received. + """ + LOGGER.debug('Issuing consumer related RPC commands') + self._channel_in.add_on_cancel_callback(self.on_consumer_cancelled) + self._consumer_tag = self._channel_in.basic_consume( + self.on_message, + self.input_queue["queue_name"], + exclusive=self.single_instance) + + def stop_consuming(self): + """ + From pika docs: + + Tell RabbitMQ that you would like to stop consuming by sending the + Basic.Cancel RPC command. + """ + if self._channel_in is not None: + LOGGER.debug('Sending a Basic.Cancel RPC command to RabbitMQ') + self._channel_in.basic_cancel(self.on_cancelok, self._consumer_tag) + else: + LOGGER.warning( + 'input queue consuming cannot be cancelled properly ' + 'because input channel is already None') + ## XXX: restart or what? + + def on_cancelok(self, unused_frame): + """ + From pika docs: + + This method is invoked by pika when RabbitMQ acknowledges the + cancellation of a consumer. At this point we will close the + channel. This will invoke the on_channel_closed method once the + channel has been closed, which will in-turn close the + connection. + + Args: + `unused_frame`: The Basic.CancelOk frame + """ + LOGGER.debug('RabbitMQ acknowledged the cancellation of the consumer') + self.close_channel("in") + + def on_consumer_cancelled(self, method_frame): + """ + From pika docs: + + Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer + receiving messages. + + Args: + `method_frame`: The Basic.Cancel frame + """ + LOGGER.info('Consumer was cancelled remotely, shutting down: %r', + method_frame) + if self._channel_in is not None: + self._channel_in.close() + else: + LOGGER.warning('input channel cannot be closed because it is already None') + ## XXX: restart or what? + + def acknowledge_message(self, delivery_tag): + """ + From pika docs: + + Acknowledge the message delivery from RabbitMQ by sending a Basic.Ack + RPC method for the delivery tag. + + Args: + `delivery_tag`: The delivery tag from the Basic.Deliver frame. + """ + LOGGER.debug('Acknowledging message %r', delivery_tag) + self._channel_in.basic_ack(delivery_tag) + + def nacknowledge_message(self, delivery_tag, reason, requeue=False): + """ + Dis-acknowledge the message delivery by sending a Nack + (RabbitMQ-specific extension of AMQP) for the delivery tag. + + Args: + `delivery_tag`: The delivery tag from the Basic.Deliver frame. + """ + ## FIXME?: maybe it should be INFO? + LOGGER.debug('Not-Acknowledging message whose delivery tag is %r\n' + 'Reason: %r\nRequeue: %r', delivery_tag, reason, requeue) + self._channel_in.basic_nack(delivery_tag, multiple=False, requeue=requeue) + + def on_message(self, channel, basic_deliver, properties, body): + """ + From pika docs: + + Invoked by pika when a message is delivered from RabbitMQ. The + channel is passed for your convenience. The basic_deliver object + that is passed in carries the exchange, routing key, delivery + tag and a redelivered flag for the message. The properties + passed in is an instance of BasicProperties with the message + properties and the body is the message that was sent. + + Args: + `channel`: The channel object. + `basic_deliver`: A pika.Spec.Basic.Deliver object. + `properties`: A pika.Spec.BasicProperties object. + `body`: The message body. + """ + exc_info = None + delivery_tag = basic_deliver.delivery_tag + routing_key = basic_deliver.routing_key + try: + LOGGER.debug('Received message #%r routed with key %r)', + delivery_tag, routing_key) + try: + self.input_callback(routing_key, body, properties) + except AuthAPICommunicationError as exc: + sys.exit(exc) + except Exception as exc: + # Note: catching Exception is OK here. We *do* want to + # catch any exception, except SystemExit, KeyboardInterrupt etc. + event_info_msg = self._get_error_event_info_msg(exc, properties) + LOGGER.error('Exception occured while processing message #%r%s ' + '[%s: %r]. The message will be nack-ed...', + delivery_tag, + (' ({0})'.format(event_info_msg) if event_info_msg + else ''), + type(exc).__name__, + getattr(exc, 'args', exc), + exc_info=True) + LOGGER.debug('Metadata of message ' ## FIXME?: maybe it should be INFO? + '#%r:\nrouting key: %r\nproperties: %r', + delivery_tag, routing_key, properties) + LOGGER.debug('Body of message #%r:\n%r', delivery_tag, body) + self.nacknowledge_message(delivery_tag, '{0!r} in {1!r}'.format(type(exc), self)) + except: + # we do want to nack and requeue event on SystemExit, KeyboardInterrupt etc. + exc_info = sys.exc_info() + LOGGER.info('%r occured while processing message #%r. ' + 'The message will be requeued...', + exc_info[1], + delivery_tag) + self.nacknowledge_message(delivery_tag, '{0!r} in {1!r}'.format(exc_info[1], self), + requeue=True) + # now we can re-raise the original exception + raise exc_info[0], exc_info[1], exc_info[2] + else: + self.acknowledge_message(delivery_tag) + finally: + del exc_info + + def input_callback(self, routing_key, body, properties): + """ + Placeholder for input_callback defined by child classes. + + Args: + `routing_key`: + The routing key to send the message with. + `body`: + The body of the message. + `properties`: + A pika.BasicProperties instance (which, among others, has + the `headers` attribute -- being None or a dict of custom + message headers). + """ + raise NotImplementedError + + @staticmethod + @contextlib.contextmanager + def setting_error_event_info(rid_or_record_dict): + """ + Set error event info (rid and optionally id) on any raised Exception. + + (For better error inspection when analyzing logs...) + + Args: + `rid_or_record_dict`: + Either a message rid (a string) or an event data + (a RecordDict instance or just a dict). + + To be used in subclasses in input_callback or methods called by it. + Some simple examples: + + event_rid = + with self.setting_error_event_info(event_rid): + + + event_record_dict = + with self.setting_error_event_info(event_record_dict): + + """ + if isinstance(rid_or_record_dict, (basestring, types.NoneType)): + event_rid = rid_or_record_dict + event_id = None + else: + event_rid = rid_or_record_dict.get('rid') + event_id = rid_or_record_dict.get('id') + + try: + yield + except Exception as exc: + # see also: _get_error_event_info_msg() above + if getattr(exc, '_n6_event_rid', None) is None: + exc._n6_event_rid = event_rid + if getattr(exc, '_n6_event_id', None) is None: + exc._n6_event_id = event_id + raise + + @staticmethod + def _get_error_event_info_msg(exc, properties): + # see also: setting_error_event_info() below + msg_parts = [] + message_id = getattr(properties, 'message_id', None) + event_rid = getattr(exc, '_n6_event_rid', None) + event_id = getattr(exc, '_n6_event_id', None) + # (note: message_id is often the same as rid) + if message_id is not None and (event_rid is None or message_id != event_rid): + msg_parts.append('AMQP message_id: {0}'.format(message_id)) + if event_rid is not None: + msg_parts.append('event rid: {0}'.format(event_rid)) + if event_id is not None: + msg_parts.append('event id: {0}'.format(event_id)) + return ', '.join(msg_parts) + + # Output-exchanges-related stuff: + + def setup_output_exchanges(self): + """ + Setup all output exchanges on RabbitMQ (by invoking Exchange.Declare + RPC commands for each of them0. When such a command is + completed, the on_output_exchange_declared method (with the + `exchange` argument already bound using functools.partial) will + be invoked by pika. + """ + for params in self.output_queue: + exchange = params["exchange"] + callback = functools.partial(self.on_output_exchange_declared, exchange) + LOGGER.debug('Declaring output exchange %r', exchange) + self._channel_out.exchange_declare( + callback, + exchange, + params["exchange_type"], + durable=True) + + def on_output_exchange_declared(self, exchange, frame): + """ + Args: + `exchange`: Declared exchange name. + `frame`: Exchange.DeclareOk response frame. + """ + LOGGER.debug('Output exchange %r declared', exchange) + self._declared_output_exchanges.add(exchange) + if self._channel_out is not None: + assert frame.channel_number == self._channel_out.channel_number + assert isinstance(self.output_queue, list) + if len(self._declared_output_exchanges) == len(self.output_queue): + LOGGER.debug('All output exchanges declared') + self.output_ready = True + self.start_publishing() + else: + LOGGER.error('Cannot set up publishing because output channel is None') + ## XXX: restart or what? + + def start_publishing(self): + """ + Placeholder for the publishing method. + + It gets called after all output exchanges have been declared and + are ready. Publishers should override this method. + """ + + def publish_output(self, routing_key, body, prop_kwargs=None, exchange=None): + """ + Publish to the (default or specified) output exchange. + + Args: + `routing_key`: + The routing key to send the message with. + `body`: + The body of the message. + `prop_kwargs` (optional): + Custom keyword arguments for pika.BasicProperties. + `exchange` (optional): + The exchange name. If omitted, the 'exchange' value of + the first item of the `output_queue` instance attribute + will be used. + """ + if self._closing: + # CRITICAL because for a long time (since 2013-04-26!) there was a silent return here! + LOGGER.critical('Trying to publish when the `_closing` flag is true!') + raise RuntimeError('trying to publish when the `_closing` flag is true!') + if not self.output_ready: + # CRITICAL because for a long time (since 2013-04-26!) there was a silent return here! + LOGGER.critical('Trying to publish when the `output_ready` flag is false!') + raise RuntimeError('trying to publish when the `output_ready` flag is false!') + + if exchange is None: + exchange = self.output_queue[0]['exchange'] + if exchange not in self._declared_output_exchanges: + raise RuntimeError('exchange {0!r} has not been declared'.format(exchange)) + + kwargs_for_properties = self.basic_prop_kwargs.copy() + if prop_kwargs is not None: + kwargs_for_properties.update(prop_kwargs) + if 'headers' in kwargs_for_properties and ( + not kwargs_for_properties['headers']): + # delete empty `headers` dict + del kwargs_for_properties['headers'] + properties = pika.BasicProperties(**kwargs_for_properties) + + self.basic_publish(exchange=exchange, + routing_key=routing_key, + body=body, + properties=properties) + + # basic_publish() might trigger the on_connection_closed() callback + if self._closing or not self.output_ready: + raise n6AMQPCommunicationError( + 'after output channel\'s basic_publish(): _closing={0!r} ' + '(should be False) and output_ready={1!r} (should be True) ' + '-- which means that AMQP communication is no longer possible ' + 'and most probably the data have not been sent ' + '(routing key: {2!r}, body length: {3})'.format( + self._closing, self.output_ready, routing_key, len(body))) + + def basic_publish(self, exchange, routing_key, body, properties): + """ + Thin wrapper around pika's basic_publish -- for easier testing/mocking. + + Typically it is *not* used directly but only by calling the + publish_output() method. + """ + LOGGER.debug('Publishing message to %r, rk: %r\n' + 'Properties: %s\nBody: %r', + exchange, + routing_key, + pprint.pformat(properties), + body) + self._channel_out.basic_publish(exchange=exchange, + routing_key=routing_key, + body=body, + properties=properties) diff --git a/N6Core/n6/collectors/__init__.py b/N6Core/n6/collectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/N6Core/n6/collectors/abuse_ch.py b/N6Core/n6/collectors/abuse_ch.py new file mode 100644 index 0000000..fa31dee --- /dev/null +++ b/N6Core/n6/collectors/abuse_ch.py @@ -0,0 +1,433 @@ +# Copyright (c) 2013-2018 NASK. All rights reserved. + +""" +Collectors: abuse-ch.spyeye-doms, abuse-ch.spyeye-ips, +abuse-ch.zeus-doms, abuse-ch.zeus-ips, abuse-ch.zeustracker, +abuse-ch.palevo-doms, abuse-ch.palevo-ips, abuse-ch.feodotracker, +abuse-ch.ransomware, abuse-ch.ssl-blacklist, abuse-ch.ssl-blacklist-dyre +""" + +import csv +import json +import re +import sys +from collections import MutableMapping +from cStringIO import StringIO +from datetime import datetime + +from lxml import html +from lxml.etree import ParserError, XMLSyntaxError + +from n6.collectors.generic import ( + BaseRSSCollector, + BaseUrlDownloaderCollector, + BaseOneShotCollector, + CollectorWithStateMixin, + entry_point_factory, +) +from n6lib.log_helpers import get_logger + + +LOGGER = get_logger(__name__) + + +class NoNewDataException(Exception): + + """ + Exception raised when the source does not provide any new data. + """ + + +class _BaseAbuseChMixin(object): + + raw_format_version_tag = '201406' + type = 'blacklist' + + def process_data(self, data): + return data + + +class AbuseChSpyeyeDomsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_spyeye_doms" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "spyeye-doms" + + +class AbuseChSpyeyeIpsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_spyeye_ips" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "spyeye-ips" + + +class AbuseChZeusDomsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_zeus_doms" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "zeus-doms" + + +class AbuseChZeusIpsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_zeus_ips" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "zeus-ips" + + +class AbuseChPalevoDomsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_palevo_doms" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "palevo-doms" + + +class AbuseChPalevoIpsCollector(_BaseAbuseChMixin, BaseUrlDownloaderCollector, BaseOneShotCollector): + + config_group = "abusech_palevo_ips" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "palevo-ips" + + +class AbuseChZeusTrackerCollector(BaseRSSCollector): + + config_group = "abusech_zeustracker" + + def get_source_channel(self, **kwargs): + return 'zeustracker' + + def rss_item_to_relevant_data(self, item): + title, description = None, None + for i in item: + if i.tag == 'title': + title = i.text + elif i.tag == 'description': + description = i.text + return (title, description) + + +class AbuseChFeodoTrackerCollector(BaseRSSCollector): + + config_group = "abusech_feodotracker" + + def get_source_channel(self, **kwargs): + return 'feodotracker' + + def rss_item_to_relevant_data(self, item): + description = None + for i in item: + if i.tag == 'description': + description = i.text + return (description) + + +class AbuseChRansomwareTrackerCollector(CollectorWithStateMixin, + BaseUrlDownloaderCollector, + BaseOneShotCollector): + + type = 'file' + config_group = "abusech_ransomware" + content_type = 'text/csv' + timestamp_pattern = '%Y-%m-%d %H:%M:%S' + + def __init__(self, *args, **kwargs): + super(AbuseChRansomwareTrackerCollector, self).__init__(*args, **kwargs) + self._state = self.load_state() + if not isinstance(self._state, MutableMapping): + self._state = { + 'timestamp': None, + } + if not self._state['timestamp']: # first run of collector + self.timestamp = '1970-01-01 00:00:00' + else: + self.timestamp = self._state['timestamp'] + + def get_source_channel(self, **kwargs): + return "ransomware" + + def process_data(self, data): + output = StringIO() + writer = csv.writer(output, delimiter=',', quotechar='"') + rows = csv.reader(StringIO(data), delimiter=',', quotechar='"') + newest_entry = None + for row in rows: + if not row or row[0].startswith('#'): + continue + timestamp = datetime.strptime(row[0], self.timestamp_pattern) + if not newest_entry: + newest_entry = row[0] + if timestamp > datetime.strptime(self.timestamp, self.timestamp_pattern): + writer.writerow(row) + else: + break + if newest_entry: + self._state['timestamp'] = newest_entry + return output.getvalue() + + def start_publishing(self): + """ + Extend the method to save the date of the latest entry. + """ + super(AbuseChRansomwareTrackerCollector, self).start_publishing() + self.save_state(self._state) + + +class _AbuseChSSLBlacklistBaseCollector(CollectorWithStateMixin, BaseRSSCollector): + + """ + Base collector class for 'SSL Blacklist' and 'SSL Blacklist Dyre' + sources. + + Note that, contrary to their names, they are *event-based* sources. + """ + + # XPath to main table's records. + details_xpath = "//table[@class='tlstable']//th[text()='{field}']/following-sibling::td" + + # In order to get 'td' elements from the 'tbody' of a table only, + # select 'tr' tags NOT containing 'th' elements. LXML's parser does + # not get the exact tree, so XPath cannot search through 'tbody'. + binaries_xpath = "//table[@class='sortable']//tr[not(child::th)]" + + # The dict maps output JSON's field names to table labels. + tls_table_labels = { + 'subject': 'Subject:', + 'issuer': 'Issuer:', + 'fingerprint': 'Fingerprint (SHA1):', + 'status': 'Status:', + } + + # regex for the 'Reason' part of a 'Status' row + reason_regex = re.compile(r''' + (?:Reason:[ ]*) + (?P.*) # match a text between 'Reason:' + (?=,[ ]*Listing) # and ', Listing' + ''', re.VERBOSE) + + # regex for the 'Listing date' part of a 'Status' row + datetime_regex = re.compile(r''' + (?:Listing[ ]date:[ ]*) + (?P
\d{4}-\d{2}-\d{2}[ ] # match a date + (?:\d{2}:){2}\d{2}) # match time + ''', re.VERBOSE) + + + def __init__(self, *args, **kwargs): + super(_AbuseChSSLBlacklistBaseCollector, self).__init__(*args, **kwargs) + self._rss_feed_url = self.config['url'] + # separate timeouts for downloading detail pages + self._details_download_timeout = int(self.config.get('details_download_timeout', 12)) + self._details_retry_timeout = int(self.config.get('details_retry_timeout', 4)) + # attribute to store data created from + # detail pages, before deduplication + self._complete_data = None + + def run_handling(self): + try: + self._output_components = self.get_output_components(**self.input_data) + except NoNewDataException: + LOGGER.info('No new data from the %s source.', self.source_name) + else: + self.run() + self.save_state(self._complete_data) + LOGGER.info('Stopped') + + def get_output_data_body(self, **kwargs): + """ + Overridden method returns newly created data structure. + + Returns: + JSON object describing new and updated elements from + the RSS feed. + + Raises: + NoNewDataException: if the source provides no new data. + + Output data structure is a dict of which keys are URLs to + elements' detail pages and values are dicts containing items + extracted from those pages. + """ + old_data = self.load_state() + downloaded_rss = self._download_retry(self._rss_feed_url) + new_links = self._process_rss(downloaded_rss) + new_data = self._get_rss_items_details(new_links) + # *Copy* downloaded data before deduplication, to be saved later. + self._complete_data = dict(new_data) + if old_data: + # Get keys of a newly created dict and of a dict created + # during previous run. Keys are URLs to elements' detail + # pages. + downloaded_links = set(new_data.iterkeys()) + old_links = set(old_data.iterkeys()) + common_links = old_links & downloaded_links + # If there are any URLs common to new and previous RSS, + # there is a risk of duplication of data. + if common_links: + self._deduplicate_data(old_data, new_data, common_links) + if not new_data: + raise NoNewDataException + return json.dumps(new_data) + + def rss_item_to_relevant_data(self, item): + """ + Overridden method: create a URL to a detail page from an RSS + element. + + Args: + `item`: a single item from the RSS feed. + + Returns: + URL to item's detail page. + """ + url = None + for i in item: + if i.tag == 'link': + url = i.text + break + if url is None: + LOGGER.warning("RSS item without a link to its detail page occurred.") + return url + + def _get_rss_items_details(self, urls): + """ + Create a dict mapping elements' detail pages URLs to dicts + describing these elements. + + Args: + `urls` (list): URLs to RSS feed's elements' detail pages. + + Returns: + A dict created from fetched data. + """ + items = {} + for url in urls: + if url is None: + continue + details_page = self._download_retry_external( + url, self._details_download_timeout, self._details_retry_timeout) + if not details_page: + LOGGER.warning("Could not download details page with URL: %s", url) + continue + try: + parsed_page = html.fromstring(details_page) + except (ParserError, XMLSyntaxError): + LOGGER.warning("Could not parse event's details page with URL: %s", url) + continue + items[url] = self._get_main_details(parsed_page) + binaries_table_body = parsed_page.xpath(self.binaries_xpath) + if binaries_table_body: + items[url]['binaries'] = [tuple(x) for x in + self._get_binaries_details(binaries_table_body)] + return items + + def _get_main_details(self, parsed_page): + """ + Extract data from the main table of a detail page. + + Args: + `parsed_page` (:class:`lxml.html.HtmlElement`): + detail page after HTML parsing. + + Returns: + A dict containing items extracted from the parsed page. + """ + items = {} + for header, text_value in self.tls_table_labels.iteritems(): + table_records = parsed_page.xpath(self.details_xpath.format(field=text_value)) + if table_records and header == 'status': + status = table_records[0].text_content() + matched_datetime = self.datetime_regex.search(status) + matched_reason = self.reason_regex.search(status) + if matched_datetime: + items['timestamp'] = matched_datetime.group('dt') + if matched_reason: + items['name'] = matched_reason.group('reason') + elif table_records: + items[header] = table_records[0].text_content().strip() + return items + + def _get_binaries_details(self, table_body): + """ + Extract data from the table with associated binaries. + + Args: + `table_body` (list): 'tr' elements of the table. + + Yields: + Text content of the table's records for every binary. + """ + for tr in table_body: + yield (td.text_content().strip() for td in tr) + + def _deduplicate_data(self, old_data_body, new_data_body, common_links): + """ + Delete already published data from the output data body. + + Args: + `old_data_body` (dict): + data body from the previous run of the collector. + `new_data_body` (dict): + data body created during this run of the collector. + `common_links` (set): + URLs occurring in old and new data body. + + Returns: + New data body after deduplication process. + + The method checks elements common to previously and currently + fetched RSS feed. If there are any new associated binaries + inside of an element - it means new events can be created. + + Then it removes already published binaries records, or a whole + element - if no new binaries have been added on website. + """ + for url in common_links: + if 'binaries' not in new_data_body[url]: + new_data_body.pop(url) + elif 'binaries' in old_data_body[url]: + new_binaries = set(new_data_body[url]['binaries']) + old_binaries = set(old_data_body[url]['binaries']) + diff = new_binaries - old_binaries + if diff: + new_data_body[url]['binaries'] = list(diff) + else: + new_data_body.pop(url) + + +class AbuseChSSLBlacklistCollector(_AbuseChSSLBlacklistBaseCollector): + + config_group = "abuse_ch_ssl_blacklist" + + @property + def source_name(self): + return "Abuse.ch SSL Blacklist Collector" + + def get_source_channel(self, **kwargs): + return "ssl-blacklist" + + +class AbuseChSSLBlacklistDyreCollector(_AbuseChSSLBlacklistBaseCollector): + + config_group = "abuse_ch_ssl_blacklist_dyre" + + @property + def source_name(self): + return "Abuse.ch SSL Blacklist Collector - Dyre" + + def get_source_channel(self, **kwargs): + return "ssl-blacklist-dyre" + + +entry_point_factory(sys.modules[__name__]) diff --git a/N6Core/n6/collectors/badips.py b/N6Core/n6/collectors/badips.py new file mode 100644 index 0000000..27b5a3e --- /dev/null +++ b/N6Core/n6/collectors/badips.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2018 NASK. All rights reserved. + +""" +Collector: badips-com.server-exploit-list +""" + +import json +import sys + +from n6.collectors.generic import ( + BaseOneShotCollector, + BaseUrlDownloaderCollector, + entry_point_factory, + n6CollectorException, +) + + +class BadipsServerExploitCollector(BaseUrlDownloaderCollector, BaseOneShotCollector): + + type = 'blacklist' + content_type = 'text/csv' + config_group = "badips_server_exploit_list" + + @staticmethod + def _add_fields_name(ips, category_root, category_leaf): + formatted_ips_and_name_list = ('{};{} {} attack'.format(ip, category_leaf, category_root) + for ip in ips.rstrip('\n').split('\n')) + return '\n'.join(formatted_ips_and_name_list) + + + def get_source_channel(self, **kwargs): + return "server-exploit-list" + + def get_output_data_body(self, **kwargs): + ips_string = None + categories_json = self._download_retry(self.config['url']) + if categories_json is None: + raise n6CollectorException("Categories download failure") + categories_list = json.loads(categories_json).get('categories', []) + for category in categories_list: + if 'Parent' in category: + category_name = category.get('Name') + category_ips = self._download_retry(self.config['category_details_url'].format( + category=category_name)) + if not category_ips: + continue + formatted_category_ips = self._add_fields_name(category_ips, + category.get('Parent'), + category_name) + if ips_string is not None: + ips_string = '\n'.join([ips_string, formatted_category_ips]) + else: + ips_string = formatted_category_ips + return ips_string + + +entry_point_factory(sys.modules[__name__]) diff --git a/N6Core/n6/collectors/dns_bh.py b/N6Core/n6/collectors/dns_bh.py new file mode 100644 index 0000000..d351976 --- /dev/null +++ b/N6Core/n6/collectors/dns_bh.py @@ -0,0 +1,35 @@ +# Copyright (c) 2013-2018 NASK. All rights reserved. + +""" +Collector: dns-bh.malwaredomainscom +""" + +import sys + +from n6.collectors.generic import ( + BaseOneShotCollector, + BaseUrlDownloaderCollector, + entry_point_factory, +) +from n6lib.log_helpers import get_logger + + +LOGGER = get_logger(__name__) + + +class DnsBhMalwaredomainscomCollector(BaseUrlDownloaderCollector, BaseOneShotCollector): + + raw_format_version_tag = '201412' + + type = 'blacklist' + config_group = "malwaredomainscom" + content_type = 'text/plain' + + def get_source_channel(self, **kwargs): + return "malwaredomainscom" + + def process_data(self, data): + return data + + +entry_point_factory(sys.modules[__name__]) diff --git a/N6Core/n6/collectors/generic.py b/N6Core/n6/collectors/generic.py new file mode 100644 index 0000000..3b46b82 --- /dev/null +++ b/N6Core/n6/collectors/generic.py @@ -0,0 +1,901 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2013-2018 NASK. All rights reserved. + +""" +Collector base classes + auxiliary tools. +""" + +import cPickle +import datetime +import hashlib +import json +import os +import sys +import time +import urllib +import urllib2 + +import lxml.etree +import lxml.html + +from n6lib.config import ( + ConfigMixin, + ConfigSection, +) +from n6.base.queue import QueuedBase +from n6lib.class_helpers import all_subclasses +from n6lib.common_helpers import ascii_str +from n6lib.email_message import EmailMessage +from n6lib.log_helpers import get_logger, logging_configured + + +LOGGER = get_logger(__name__) + + +# +# Exceptions + +class n6CollectorException(Exception): + pass + + +# +# Mixin classes + +class CollectorConfigMixin(ConfigMixin): + + def set_configuration(self): + if self.is_config_spec_or_group_declared(): + self.config = self.get_config_section() + else: + # backward-compatible behavior needed by a few collectors + # that have `config_group = None` and -- at the same + # time -- no `config_spec`/`config_spec_pattern` + self.config = ConfigSection('') + + +class CollectorStateMixIn(object): + + """DO NOT USE THIS CLASS IN NEW CODE, USE ONLY CollectorWithStateMixin!""" + + _last_state = None + _current_state = None + + def __init__(self, **kwargs): + super(CollectorStateMixIn, self).__init__(**kwargs) + + + def _get_last_state(self): + self.cache_file_path = os.path.join(os.path.expanduser(self.config['cache_dir']), + self.get_cache_file_name()) + try: + with open(self.cache_file_path) as f: + self._last_state = str(f.read().strip()) + except (IOError, ValueError): + self._last_state = None + LOGGER.info("Loaded last state '%s' from cache", self._last_state) + + def _save_last_state(self): + self.cache_file_path = os.path.join(os.path.expanduser(self.config['cache_dir']), + self.get_cache_file_name()) + LOGGER.info("Saving last state '%s' to cache", self._current_state) + try: + if not os.path.isdir(os.path.expanduser(self.config['cache_dir'])): + os.makedirs(os.path.expanduser(self.config['cache_dir'])) + with open(self.cache_file_path, "w") as f: + f.write(str(self._current_state)) + except (IOError, OSError): + LOGGER.warning("Cannot save state to cache '%s'. ", self.cache_file_path) + + def get_cache_file_name(self): + return self.config['source'] + ".txt" + + +class CollectorStateMixInPlus(CollectorStateMixIn): + + """ + DO NOT USE THIS CLASS IN NEW CODE, USE ONLY CollectorWithStateMixin! + + Class for tracking state of inheriting collector. + Holds in cache dir a file with last_state variable + i.e. last processed ID or MD5, for instance. + Any type casting must be done in collector. + """ + + def get_cache_file_name(self): + return self.config['source'] + '_' + self.get_source_channel() + ".txt" + + +class CollectorWithStateMixin(object): + + """ + Mixin for tracking state of an inheriting collector. + + Any picklable object can be saved as a state and then be retrieved + as an object of the same type. + """ + + def __init__(self, *args, **kwargs): + super(CollectorWithStateMixin, self).__init__(*args, **kwargs) + self._cache_file_path = os.path.join(os.path.expanduser( + self.config['cache_dir']), self.get_cache_file_name()) + + def load_state(self): + """ + Load collector's state from cache. + + Returns: + Unpickled object of its original type. + """ + try: + with open(self._cache_file_path, 'rb') as cache_file: + state = cPickle.load(cache_file) + except (EnvironmentError, ValueError, EOFError) as exc: + LOGGER.warning( + "Could not load state, returning None (%s: %s)", + exc.__class__.__name__, + ascii_str(exc)) + state = None + else: + LOGGER.info("Loaded state: %r", state) + return state + + def save_state(self, state): + """ + Save any picklable object as a collector's state. + + Args: + `state`: a picklable object. + """ + cache_dir = os.path.dirname(self._cache_file_path) + try: + os.makedirs(cache_dir, 0700) + except OSError: + pass + with open(self._cache_file_path, 'wb') as cache_file: + cPickle.dump(state, cache_file, cPickle.HIGHEST_PROTOCOL) + LOGGER.info("Saved state: %r", state) + + def get_cache_file_name(self): + source_channel = self.get_source_channel() + source = self.get_source(source_channel=source_channel) + return '{}.{}.pickle'.format(source, self.__class__.__name__) + + +# +# Base classes + +class AbstractBaseCollector(object): + + """ + Abstract base class for a collector script implementations. + """ + + @classmethod + def get_script_init_kwargs(cls): + """ + A class method: get a dict of kwargs for instantiation in a script. + + The default implementation returns an empty dict. + """ + return {} + + # + # Permanent (daemon-like) processing + + def run_handling(self): + """ + Run the event loop until Ctrl+C is pressed. + """ + try: + self.run() + except KeyboardInterrupt: + self.stop() + + # + # Abstract methods (must be overridden) + + def run(self): + raise NotImplementedError + + def stop(self): + raise NotImplementedError + + +class BaseCollector(CollectorConfigMixin, QueuedBase, AbstractBaseCollector): + + """ + The standard "root" base class for collectors. + """ + + output_queue = { + 'exchange': 'raw', + 'exchange_type': 'topic', + } + + # None or a string being the tag of the raw data format version + # (can be set in a subclass) + raw_format_version_tag = None + + # the name of the config group + # (it does not have to be implemented if one of the `config_spec` + # or the `config_spec_pattern` attribute is set in a subclass, + # containing a declaration of exactly *one* config section) + config_group = None + + # a sequence of required config fields (can be extended in + # subclasses; typically, 'source' should be included there!) + config_required = ('source',) + # (NOTE: the `source` setting value in the config is only + # the first part -- the `label` part -- of the actual + # source specification string '