From d57571b321f9e5da7a9dbdc5f892d8989a953485 Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Mon, 4 Jan 2021 22:50:10 -0500 Subject: [PATCH] Add authenticationDatabase param to MongoTask command. Making the requirements Python 3.5 compatible. Allow passing empty username to the mongoexport command. Avoid failing on non-string values. --- exporter/mysql_query.py | 2 +- exporter/tasks.py | 3 ++- requirements/base.in | 2 ++ requirements/base.txt | 25 ++++++++++++++----------- requirements/constraints.txt | 8 ++++++++ sample-config.yaml | 11 ++++++++++- 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/exporter/mysql_query.py b/exporter/mysql_query.py index ea06d3f..55e305e 100644 --- a/exporter/mysql_query.py +++ b/exporter/mysql_query.py @@ -66,4 +66,4 @@ def _write_results_to_tsv(self, cursor, output_file): def _normalize_value(self, value): if value is None: value='NULL' - return six.text_type(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') + return six.text_type(value).replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') diff --git a/exporter/tasks.py b/exporter/tasks.py index 772b7ad..e845bb6 100644 --- a/exporter/tasks.py +++ b/exporter/tasks.py @@ -175,7 +175,8 @@ class MongoTask(Task): mongoexport --host {mongo_host} --db {mongo_db} - --username {mongo_user} + --authenticationDatabase "{mongo_auth_db}" + --username "{mongo_user}" --password "{mongo_password}" --collection {mongo_collection} --query '{query}' diff --git a/requirements/base.in b/requirements/base.in index e819381..77869e8 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -5,6 +5,8 @@ docopt edx-ccx-keys edx-opaque-keys graphitesend +mysql-connector-python +path path.py pbr pip diff --git a/requirements/base.txt b/requirements/base.txt index 41ce6f5..d84ecb8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,31 +4,34 @@ # # make upgrade # -awscli==1.18.174 # via -r requirements/base.in -botocore==1.19.14 # via awscli, s3transfer +awscli==1.18.204 # via -r requirements/base.in +botocore==1.19.44 # via awscli, s3transfer colorama==0.4.3 # via awscli docopt==0.6.2 # via -r requirements/base.in docutils==0.15.2 # via awscli edx-ccx-keys==1.1.0 # via -r requirements/base.in -edx-opaque-keys==2.1.1 # via -r requirements/base.in, edx-ccx-keys +edx-opaque-keys==2.1.0 # via -c requirements/constraints.txt, -r requirements/base.in, edx-ccx-keys graphitesend==0.10.0 # via -r requirements/base.in -importlib-metadata==2.0.0 # via stevedore +importlib-metadata==1.6.0 # via -c requirements/constraints.txt, path jmespath==0.10.0 # via botocore +more-itertools==8.6.0 # via zipp +mysql-connector-python==8.0.22 # via -r requirements/base.in path.py==12.5.0 # via -r requirements/base.in -path==15.0.0 # via path.py +path==13.1.0 # via -c requirements/constraints.txt, -r requirements/base.in, path.py pbr==5.5.1 # via -r requirements/base.in, stevedore -psutil==5.7.3 # via -r requirements/base.in +protobuf==3.14.0 # via mysql-connector-python +psutil==5.8.0 # via -r requirements/base.in pyasn1==0.4.8 # via rsa -pymongo==3.11.0 # via -r requirements/base.in, edx-opaque-keys +pymongo==3.11.2 # via -r requirements/base.in, edx-opaque-keys python-dateutil==2.8.1 # via -r requirements/base.in, botocore python-gnupg==0.4.6 # via -r requirements/base.in pyyaml==5.3.1 # via -r requirements/base.in, awscli rsa==4.5 # via awscli s3transfer==0.3.3 # via awscli -six==1.15.0 # via edx-ccx-keys, edx-opaque-keys, python-dateutil -stevedore==3.2.2 # via edx-opaque-keys -urllib3==1.25.11 # via botocore -zipp==3.4.0 # via importlib-metadata +six==1.15.0 # via edx-ccx-keys, edx-opaque-keys, protobuf, python-dateutil, stevedore +stevedore==1.32.0 # via -c requirements/constraints.txt, edx-opaque-keys +urllib3==1.26.2 # via botocore +zipp==1.0.0 # via -c requirements/constraints.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e69de29..7631f47 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -0,0 +1,8 @@ +# edx-platform Juniper version compatible. +edx-opaque-keys==2.1.0 + +# Python 3.5 compatible. +path==13.1.0 +stevedore==1.32.0 +importlib-metadata==1.6.0 +zipp==1.0.0 \ No newline at end of file diff --git a/sample-config.yaml b/sample-config.yaml index b388566..67e8bac 100644 --- a/sample-config.yaml +++ b/sample-config.yaml @@ -16,8 +16,9 @@ defaults: django_cms_settings: cms-django-settings e.g. cms.envs.devstack_docker django_database: the_django_database mongo_collection: name-of-the-mongo-collection - sql_user: user-name + sql_user: mysql-username sql_db: name-of-the-edxapp-db e.g. edxapp + sql_password: mysql-password secret_key: '' monitor: false graphite_host: localhost @@ -30,8 +31,13 @@ environments: django_config: /prod/django/config mongo_host: 0.0.0.0:9999 mongo_db: the_mongo_db + mongo_user: the_mongo_username + mongo_password: the_mongo_password + mongo_auth_db: authentication-database-name name: prod-mongodb sql_host: prod-sql-host.example.com + lms_config: path-to-the-lms-yml-config-file + studio_config: path-to-the-studio-yml-config-file edge: django_config: /edge/django/config mongo_host: 0.0.0.0:8888 @@ -44,8 +50,11 @@ environments: mongo_db: edxapp mongo_user: edxapp mongo_password: password + mongo_auth_db: authentication-database-name name: local-mongodb This name will be used to create the Mongo DB files. sql_host: edx.devstack.mysql + lms_config: path-to-the-lms-yml-config-file + studio_config: path-to-the-studio-yml-config-file organizations: edX: