From 0fa8e8387cfbcca21380bdbac1071b593cfdfd7a Mon Sep 17 00:00:00 2001 From: Aarif Date: Thu, 4 Jun 2020 16:32:19 +0500 Subject: [PATCH 1/2] Add support for python 3.7 and drop python 2 support --- .travis.yml | 8 ++-- AUTHORS | 29 ++++++++---- Makefile | 12 ++++- exporter/config.py | 7 +-- exporter/course_export.py | 9 ++-- exporter/main.py | 7 ++- exporter/mysql_query.py | 3 +- exporter/properties.py | 2 +- exporter/tasks.py | 45 ++++++++++++++---- exporter/tests/test_main.py | 12 +++-- exporter/util.py | 9 ++-- requirements.txt | 21 --------- requirements/base.in | 15 ++++++ requirements/base.txt | 34 ++++++++++++++ requirements/constraints.txt | 0 .../github.in | 2 + requirements/github_requirements.txt | 7 +++ requirements/pip-tools.in | 5 ++ requirements/pip-tools.txt | 12 +++++ requirements/test.in | 10 ++++ requirements/test.txt | 46 +++++++++++++++++++ requirements/tox.in | 3 ++ requirements/tox.txt | 19 ++++++++ test_requirements.txt | 2 - tox.ini | 5 +- 25 files changed, 250 insertions(+), 74 deletions(-) delete mode 100644 requirements.txt create mode 100644 requirements/base.in create mode 100644 requirements/base.txt create mode 100644 requirements/constraints.txt rename github_requirements.txt => requirements/github.in (80%) create mode 100644 requirements/github_requirements.txt create mode 100644 requirements/pip-tools.in create mode 100644 requirements/pip-tools.txt create mode 100644 requirements/test.in create mode 100644 requirements/test.txt create mode 100644 requirements/tox.in create mode 100644 requirements/tox.txt delete mode 100644 test_requirements.txt diff --git a/.travis.yml b/.travis.yml index c44c149..e44aa9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,8 @@ # Config file for automatic testing at travis-ci.org language: python -matrix: - include: - - python: 2.7 - env: TOXENV=py27 +python: + - 3.7 cache: - pip @@ -13,7 +11,7 @@ before_install: - pip install --upgrade pip install: - - pip install -r test_requirements.txt + - pip install -r requirements/tox.txt script: - tox diff --git a/AUTHORS b/AUTHORS index 0622a91..421c19c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,12 +1,21 @@ -Carlos Andrés Rocha -John Jarvis -Isaac Chuang -Ed Zarecor +Aarif +Abdul Mannan +Alex Dusenbery +Alex Dusenbery +Andrew Zafft +Brian Beggs +Brian Beggs Brian Wilson -Gabe Mulley -Greg Price +Dennis Jen +Diana Huang Diana Huang -Andy Armstrong -Will Daly -Eric Fischer -Dennis Jen +Feanil Patel +Gabe Mulley +Gabe Mulley +Gregory Martin +Gregory Martin +Hassan +Hassan Javeed +Stu Young +Victor Shnayder +brianhw diff --git a/Makefile b/Makefile index e672b4a..72b363e 100644 --- a/Makefile +++ b/Makefile @@ -3,4 +3,14 @@ clean: rm -rf exporter/tests/__pycache__/ test: - py.test + pytest + + +upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade +upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in + pip install -qr requirements/pip-tools.txt + pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + pip-compile --upgrade -o requirements/base.txt requirements/base.in + pip-compile --upgrade -o requirements/test.txt requirements/test.in + pip-compile --upgrade -o requirements/github_requirements.txt requirements/github.in + pip-compile --upgrade -o requirements/tox.txt requirements/tox.in diff --git a/exporter/config.py b/exporter/config.py index 64faad5..96a7db8 100644 --- a/exporter/config.py +++ b/exporter/config.py @@ -10,6 +10,7 @@ import yaml from exporter.util import merge, filter_keys +import six WORK_SUBDIR = 'course-data' @@ -59,7 +60,7 @@ def update_config(config, program_options): def merge_program_options(config, program_options): # get program options, removing '--' and replacing '-' with '_' options = {k[2:].replace('-', '_'): v for k, v - in program_options.iteritems() + in six.iteritems(program_options) if k.startswith('--')} config['options'] = options @@ -104,7 +105,7 @@ def update_environments(config): for env in ['prod', 'edge']: if env in environments: data = environments.get(env, {}) - for config_name, token_name in field_map.iteritems(): + for config_name, token_name in six.iteritems(field_map): data[config_name] = tokens.get(token_name) # different settings for edge @@ -123,7 +124,7 @@ def update_organizations(config): # lowercase orgs before selection organizations = {org.lower(): values for org, values - in config['organizations'].iteritems()} + in six.iteritems(config['organizations'])} # select only organizations in arguments organizations = filter_keys(organizations, values.get('org')) diff --git a/exporter/course_export.py b/exporter/course_export.py index d863fb1..bb8e9f3 100755 --- a/exporter/course_export.py +++ b/exporter/course_export.py @@ -49,6 +49,7 @@ from exporter.main import run_tasks, archive_directory, upload_data, get_all_courses, _get_selected_tasks from exporter.config import setup, get_config_for_env, get_config_for_course from exporter.util import make_temp_directory, with_temp_directory, merge +import six log = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def get_filename_safe_course_id(course_id, replacement_char='_'): """ try: course_key = CourseKey.from_string(course_id) - filename = unicode(replacement_char).join([course_key.org, course_key.course, course_key.run]) + filename = six.text_type(replacement_char).join([course_key.org, course_key.course, course_key.run]) except InvalidKeyError: # If the course_id doesn't parse, we will still return a value here. filename = course_id @@ -158,8 +159,4 @@ def get_filename_safe_course_id(course_id, replacement_char='_'): # The safest characters are A-Z, a-z, 0-9, , and . # We represent the first four with \w. # TODO: Once we support courses with unicode characters, we will need to revisit this. - return re.sub(r'[^\w\.\-]', unicode(replacement_char), filename) - -if __name__ == '__main__': - import sys - sys.exit(main()) + return re.sub(r'[^\w\.\-]', six.text_type(replacement_char), filename) diff --git a/exporter/main.py b/exporter/main.py index 3f367bc..cbd7135 100755 --- a/exporter/main.py +++ b/exporter/main.py @@ -63,11 +63,10 @@ from exporter.util import make_temp_directory, with_temp_directory from exporter.util import filter_keys, memoize, execute_shell from exporter.util import logging_streams_on_failure - +import six log = logging.getLogger(__name__) - # pylint: disable=missing-docstring @@ -77,7 +76,6 @@ def main(argv=None): general_config = setup(__doc__, argv=argv) for organization in general_config['organizations']: - config = get_config_for_org(general_config, organization) with make_org_directory(config, organization) as destination: @@ -304,12 +302,13 @@ def match(course): return [course for course in courses if match(course)] + def get_all_courses(**kwargs): log.info('Retrieving all courses') # make a set of fixed arguments, so we can memoize kwargs = { - k: v for k, v in kwargs.iteritems() + k: v for k, v in six.iteritems(kwargs) if k.startswith('django') or k == 'lms_config' or k == 'studio_config' } kwargs['dry_run'] = False # always query for course names diff --git a/exporter/mysql_query.py b/exporter/mysql_query.py index 5b89b6d..ea06d3f 100644 --- a/exporter/mysql_query.py +++ b/exporter/mysql_query.py @@ -2,6 +2,7 @@ import json import csv import mysql.connector +import six MAX_FETCH_SIZE = 10000 @@ -65,4 +66,4 @@ def _write_results_to_tsv(self, cursor, output_file): def _normalize_value(self, value): if value is None: value='NULL' - return unicode(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') + return six.text_type(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') diff --git a/exporter/properties.py b/exporter/properties.py index d9c6c63..641d5ef 100755 --- a/exporter/properties.py +++ b/exporter/properties.py @@ -55,7 +55,7 @@ def export_properties(config, directory, files=None, orgs=None, prefix=''): recreate_directory(directory) orgs = [o.lower() for o in orgs.split()] if orgs else ['*'] - print orgs + print(orgs) files_data = load_files(files) diff --git a/exporter/tasks.py b/exporter/tasks.py index 82ce786..772b7ad 100644 --- a/exporter/tasks.py +++ b/exporter/tasks.py @@ -11,10 +11,8 @@ from exporter.mysql_query import MysqlDumpQueryToTSV from exporter.util import NotSet, execute_shell - setup_logging() - log = logging.getLogger(__name__) MAX_TRIES_FOR_MARKER_FILE_CHECK = 5 @@ -84,6 +82,7 @@ def write_failed_file(cls, **kwargs): class OrgTask(FilenameMixin): """ Mixin class for organization level tasks.""" + @staticmethod def entity_name(kwargs): organization = _substitute_non_ascii_chars(kwargs['organization']) @@ -142,9 +141,10 @@ def run(cls, filename, dry_run, **kwargs): log.debug(query) if dry_run: - print 'SQL: {0}'.format(query) + print('SQL: {0}'.format(query)) else: - mysql_query = MysqlDumpQueryToTSV(kwargs.get('sql_host'), kwargs.get('sql_user'), kwargs.get('sql_password'), kwargs.get('sql_db'), filename) + mysql_query = MysqlDumpQueryToTSV(kwargs.get('sql_host'), kwargs.get('sql_user'), + kwargs.get('sql_password'), kwargs.get('sql_db'), filename) mysql_query.execute(query) @classmethod @@ -195,7 +195,7 @@ def run(cls, filename, dry_run, **kwargs): cmd = cmd.format(filename=filename, query=query, **kwargs) if dry_run: - print 'MONGO: {0}'.format(query) + print('MONGO: {0}'.format(query)) else: execute_shell(cmd, **kwargs) @@ -267,9 +267,9 @@ def run(cls, filename, dry_run, **kwargs): ) if dry_run: - print 'Copy S3 File: {0} to {1}'.format( + print('Copy S3 File: {0} to {1}'.format( s3_source_filename, - filename) + filename)) else: # First check to see that the export data was successfully generated # by looking for a marker file for that run. Return a more severe failure, @@ -378,6 +378,7 @@ class CourseEnrollmentTask(CourseTask, SQLTask): WHERE course_id='{course}' """ + class CourseGradesTask(CourseTask, SQLTask): NAME = 'grades_persistentcoursegrade' SQL = """ @@ -394,6 +395,7 @@ class CourseGradesTask(CourseTask, SQLTask): ORDER BY grades_persistentcoursegrade.user_id """ + class SubsectionGradesTask(CourseTask, SQLTask): NAME = 'grades_persistentsubsectiongrade' SQL = """ @@ -413,6 +415,7 @@ class SubsectionGradesTask(CourseTask, SQLTask): grades_persistentsubsectiongrade.first_attempted """ + class GeneratedCertificateTask(CourseTask, SQLTask): NAME = 'certificates_generatedcertificate' SQL = """ @@ -492,6 +495,7 @@ class StudentLanguageProficiencyTask(CourseTask, SQLTask): class CourseWikiTask(CourseTask): """ Mixin for Course Wiki related tasks """ + @classmethod def run(cls, filename, dry_run, **kwargs): course_key = CourseKey.from_string(kwargs['course']) @@ -613,6 +617,7 @@ class AssessmentAIClassifierTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class AssessmentAIClassifierSetTask(ORA2CourseTask, SQLTask): NAME = 'assessment_aiclassifierset' SQL = """ @@ -620,6 +625,7 @@ class AssessmentAIClassifierSetTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + # Not used class AssessmentAIGradingWorkflowTask(ORA2CourseTask, SQLTask): NAME = 'assessment_aigradingworkflow' @@ -628,6 +634,7 @@ class AssessmentAIGradingWorkflowTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class AssessmentAITrainingWorkflowTask(ORA2CourseTask, SQLTask): NAME = 'assessment_aitrainingworkflow' SQL = """ @@ -635,6 +642,7 @@ class AssessmentAITrainingWorkflowTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class AssessmentAITrainingWorkflowTrainingExamplesTask(ORA2CourseTask, SQLTask): NAME = 'assessment_aitrainingworkflow_training_examples' SQL = """ @@ -643,6 +651,7 @@ class AssessmentAITrainingWorkflowTrainingExamplesTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class AssessmentAssessmentTask(ORA2CourseTask, SQLTask): NAME = 'assessment_assessment' SQL = """ @@ -652,6 +661,7 @@ class AssessmentAssessmentTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentAssessmentFeedbackTask(ORA2CourseTask, SQLTask): NAME = 'assessment_assessmentfeedback' SQL = """ @@ -664,6 +674,7 @@ class AssessmentAssessmentFeedbackTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentAssessmentFeedbackAssessmentsTask(ORA2CourseTask, SQLTask): NAME = 'assessment_assessmentfeedback_assessments' SQL = """ @@ -674,6 +685,7 @@ class AssessmentAssessmentFeedbackAssessmentsTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentAssessmentFeedbackOptionsTask(ORA2CourseTask, SQLTask): """ Note the 's' in FeedbackOptions (as compared to below) @@ -690,6 +702,7 @@ class AssessmentAssessmentFeedbackOptionsTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentAssessmentFeedbackOptionTask(ORA2CourseTask, SQLTask): """ Note the lack of 's' in FeedbackOption (as compared to above) @@ -708,6 +721,7 @@ class AssessmentAssessmentFeedbackOptionTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentAssessmentPartTask(ORA2CourseTask, SQLTask): NAME = 'assessment_assessmentpart' SQL = """ @@ -718,6 +732,7 @@ class AssessmentAssessmentPartTask(ORA2CourseTask, SQLTask): WHERE si.course_id="{course}" """ + class AssessmentCriterionTask(ORA2CourseTask, SQLTask): NAME = 'assessment_criterion' SQL = """ @@ -746,6 +761,7 @@ class AssessmentCriterionTask(ORA2CourseTask, SQLTask): WHERE acs.course_id="{course}") """ + class AssessmentCriterionOptionTask(ORA2CourseTask, SQLTask): NAME = 'assessment_criterionoption' SQL = """ @@ -776,6 +792,7 @@ class AssessmentCriterionOptionTask(ORA2CourseTask, SQLTask): WHERE acs.course_id="{course}")) """ + class AssessmentPeerWorkflowTask(ORA2CourseTask, SQLTask): NAME = 'assessment_peerworkflow' SQL = """ @@ -783,6 +800,7 @@ class AssessmentPeerWorkflowTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class AssessmentPeerWorkflowItemTask(ORA2CourseTask, SQLTask): NAME = 'assessment_peerworkflowitem' SQL = """ @@ -791,6 +809,7 @@ class AssessmentPeerWorkflowItemTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class AssessmentRubricTask(ORA2CourseTask, SQLTask): """ There can be rubrics for assessments, training examples, AI Grading Workflows, @@ -821,6 +840,7 @@ class AssessmentRubricTask(ORA2CourseTask, SQLTask): WHERE acs.course_id="{course}" """ + class AssessmentStudentTrainingWorkflow(ORA2CourseTask, SQLTask): NAME = 'assessment_studenttrainingworkflow' SQL = """ @@ -828,6 +848,7 @@ class AssessmentStudentTrainingWorkflow(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class AssessmentStudentTrainingWorkflowItemTask(ORA2CourseTask, SQLTask): NAME = 'assessment_studenttrainingworkflowitem' SQL = """ @@ -857,6 +878,7 @@ class AssessmentTrainingExampleTask(ORA2CourseTask, SQLTask): WHERE stw.course_id="{course}" """ + class AssessmentTrainingExampleOptionsSelectedTask(ORA2CourseTask, SQLTask): NAME = 'assessment_trainingexample_options_selected' SQL = """ @@ -876,6 +898,7 @@ class AssessmentTrainingExampleOptionsSelectedTask(ORA2CourseTask, SQLTask): WHERE stw.course_id="{course}") """ + class SubmissionsScoreTask(ORA2CourseTask, SQLTask): NAME = 'submissions_score' SQL = """ @@ -884,6 +907,7 @@ class SubmissionsScoreTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class SubmissionsScoreSummaryTask(ORA2CourseTask, SQLTask): NAME = 'submissions_scoresummary' SQL = """ @@ -892,6 +916,7 @@ class SubmissionsScoreSummaryTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class SubmissionsStudentItemTask(ORA2CourseTask, SQLTask): NAME = 'submissions_studentitem' SQL = """ @@ -899,6 +924,7 @@ class SubmissionsStudentItemTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class SubmissionsSubmissionTask(ORA2CourseTask, SQLTask): NAME = 'submissions_submission' SQL = """ @@ -907,6 +933,7 @@ class SubmissionsSubmissionTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + class WorkflowAssessmentWorkflowTask(ORA2CourseTask, SQLTask): NAME = 'workflow_assessmentworkflow' SQL = """ @@ -914,6 +941,7 @@ class WorkflowAssessmentWorkflowTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}" """ + class WorkflowAssessmentWorkflowStepTask(ORA2CourseTask, SQLTask): NAME = 'workflow_assessmentworkflowstep' SQL = """ @@ -922,6 +950,7 @@ class WorkflowAssessmentWorkflowStepTask(ORA2CourseTask, SQLTask): WHERE course_id="{course}") """ + # End ORA2 Tables ================== class ForumsTask(CourseTask, MongoTask): @@ -997,7 +1026,7 @@ def run(cls, filename, dry_run, **kwargs): organizations = [kwargs['organization']] + kwargs.get('other_names', []) kwargs['comma_sep_courses'] = ','.join(kwargs['courses']) kwargs['all_organizations'] = ' '.join(organizations) - kwargs['max_tries'] = 3 # always retry this task a couple of times. + kwargs['max_tries'] = 3 # always retry this task a couple of times. return super(OrgEmailOptInTask, cls).run(filename, dry_run, **kwargs) diff --git a/exporter/tests/test_main.py b/exporter/tests/test_main.py index e8dff9b..c12e71f 100644 --- a/exporter/tests/test_main.py +++ b/exporter/tests/test_main.py @@ -1,3 +1,6 @@ + +import operator + import mock from exporter import main, tasks @@ -14,8 +17,8 @@ def test_get_selected_tasks_no_options_org_tasks(): def test_get_selected_tasks_no_options_course_tasks(): assert sorted([ - task for task in tasks.DEFAULT_TASKS if issubclass(task, tasks.CourseTask) - ]) == sorted(main._get_selected_tasks(tasks.CourseTask, [], [])) + task for task in tasks.DEFAULT_TASKS if issubclass(task, tasks.CourseTask)], key=operator.attrgetter('NAME')) \ + == sorted(main._get_selected_tasks(tasks.CourseTask, [], []), key=operator.attrgetter('NAME')) def test_get_selected_tasks_specified_from_options(): @@ -23,9 +26,8 @@ def test_get_selected_tasks_specified_from_options(): def test_get_selected_tasks_excluded_tasks(): - assert sorted( - set(tasks.DEFAULT_TASKS) - set([tasks.OrgEmailOptInTask]) - ) == sorted(main._get_selected_tasks(tasks.Task, [], ['OrgEmailOptInTask'])) + assert sorted(set(tasks.DEFAULT_TASKS) - set([tasks.OrgEmailOptInTask]), key=operator.attrgetter('NAME')) \ + == sorted(main._get_selected_tasks(tasks.Task, [], ['OrgEmailOptInTask']), key=operator.attrgetter('NAME')) def test_run_tasks_happy_path(): diff --git a/exporter/util.py b/exporter/util.py index 824b5ad..39a68db 100644 --- a/exporter/util.py +++ b/exporter/util.py @@ -9,6 +9,7 @@ import subprocess import tempfile import time +import six log = logging.getLogger(__name__) @@ -20,8 +21,8 @@ def __str__(cls): raise ValueError("Value not set in child class") -class NotSet(object): - __metaclass__ = MetaNotSet +class NotSet(six.with_metaclass(MetaNotSet, object)): + pass # Dictionary utilities @@ -51,7 +52,7 @@ def filter_keys(mapping, keys): if keys: result = {k: {} for k in keys} result.update({k: v for k, v - in mapping.iteritems() + in six.iteritems(mapping) if k in keys}) else: result = mapping.copy() @@ -100,7 +101,7 @@ def with_temp_directory(*d_args, **d_kwargs): def wrap(func): @functools.wraps(func) def wrapped(*args): - if func.func_code.co_argcount - len(args) == 1: + if func.__code__.co_argcount - len(args) == 1: with make_temp_directory(*d_args, **d_kwargs) as temp_dir: args += (temp_dir,) return func(*args) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9375d18..0000000 --- a/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Application -awscli -docopt -edx-ccx-keys -edx-opaque-keys -graphitesend==0.10.0 # Apache -mysql-connector-python==8.0.18 -path.py -pbr -pip==1.5.6 # MIT // AN-4322 -pymongo -python-dateutil -python-gnupg==0.3.6 -pyyaml -psutil - -# Tests -coverage==4.3.4 -mock==2.0.0 -pytest==3.0.7 -pytest-cov==2.4.0 diff --git a/requirements/base.in b/requirements/base.in new file mode 100644 index 0000000..e819381 --- /dev/null +++ b/requirements/base.in @@ -0,0 +1,15 @@ +-c constraints.txt + +awscli +docopt +edx-ccx-keys +edx-opaque-keys +graphitesend +path.py +pbr +pip +pymongo +python-dateutil +python-gnupg +pyyaml +psutil diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000..41ce6f5 --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +awscli==1.18.174 # via -r requirements/base.in +botocore==1.19.14 # via awscli, s3transfer +colorama==0.4.3 # via awscli +docopt==0.6.2 # via -r requirements/base.in +docutils==0.15.2 # via awscli +edx-ccx-keys==1.1.0 # via -r requirements/base.in +edx-opaque-keys==2.1.1 # via -r requirements/base.in, edx-ccx-keys +graphitesend==0.10.0 # via -r requirements/base.in +importlib-metadata==2.0.0 # via stevedore +jmespath==0.10.0 # via botocore +path.py==12.5.0 # via -r requirements/base.in +path==15.0.0 # via path.py +pbr==5.5.1 # via -r requirements/base.in, stevedore +psutil==5.7.3 # via -r requirements/base.in +pyasn1==0.4.8 # via rsa +pymongo==3.11.0 # via -r requirements/base.in, edx-opaque-keys +python-dateutil==2.8.1 # via -r requirements/base.in, botocore +python-gnupg==0.4.6 # via -r requirements/base.in +pyyaml==5.3.1 # via -r requirements/base.in, awscli +rsa==4.5 # via awscli +s3transfer==0.3.3 # via awscli +six==1.15.0 # via edx-ccx-keys, edx-opaque-keys, python-dateutil +stevedore==3.2.2 # via edx-opaque-keys +urllib3==1.25.11 # via botocore +zipp==3.4.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements/constraints.txt b/requirements/constraints.txt new file mode 100644 index 0000000..e69de29 diff --git a/github_requirements.txt b/requirements/github.in similarity index 80% rename from github_requirements.txt rename to requirements/github.in index c1594ce..98ec1f8 100644 --- a/github_requirements.txt +++ b/requirements/github.in @@ -1 +1,3 @@ +-c constraints.txt + http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-1.2.2.zip diff --git a/requirements/github_requirements.txt b/requirements/github_requirements.txt new file mode 100644 index 0000000..f6ba802 --- /dev/null +++ b/requirements/github_requirements.txt @@ -0,0 +1,7 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-1.2.2.zip # via -r requirements/github.in diff --git a/requirements/pip-tools.in b/requirements/pip-tools.in new file mode 100644 index 0000000..582243f --- /dev/null +++ b/requirements/pip-tools.in @@ -0,0 +1,5 @@ +# Allows us to generate version pins with `pip-compile`, and install +# them deterministically with `pip-sync`. +# `pip-compile` itself is used to generate pip-tools.txt. + +pip-tools \ No newline at end of file diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt new file mode 100644 index 0000000..4b50eae --- /dev/null +++ b/requirements/pip-tools.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +click==7.1.2 # via pip-tools +pip-tools==5.3.1 # via -r requirements/pip-tools.in +six==1.15.0 # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements/test.in b/requirements/test.in new file mode 100644 index 0000000..4e95484 --- /dev/null +++ b/requirements/test.in @@ -0,0 +1,10 @@ +-c constraints.txt + +-r base.txt +-r github_requirements.txt + +coverage +mock +pytest +pytest-cov +setuptools diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000..eea3715 --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,46 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +attrs==20.3.0 # via pytest +awscli==1.18.174 # via -r requirements/base.txt +botocore==1.19.14 # via -r requirements/base.txt, awscli, s3transfer +colorama==0.4.3 # via -r requirements/base.txt, awscli +coverage==5.3 # via -r requirements/test.in, pytest-cov +docopt==0.6.2 # via -r requirements/base.txt +docutils==0.15.2 # via -r requirements/base.txt, awscli +edx-ccx-keys==1.1.0 # via -r requirements/base.txt +edx-opaque-keys==2.1.1 # via -r requirements/base.txt, edx-ccx-keys +graphitesend==0.10.0 # via -r requirements/base.txt +importlib-metadata==2.0.0 # via -r requirements/base.txt, pluggy, pytest, stevedore +iniconfig==1.1.1 # via pytest +jmespath==0.10.0 # via -r requirements/base.txt, botocore +mock==4.0.2 # via -r requirements/test.in +http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-1.2.2.zip # via -r requirements/github.txt +packaging==20.4 # via pytest +path.py==12.5.0 # via -r requirements/base.txt +path==15.0.0 # via -r requirements/base.txt, path.py +pbr==5.5.1 # via -r requirements/base.txt, stevedore +pluggy==0.13.1 # via pytest +psutil==5.7.3 # via -r requirements/base.txt +py==1.9.0 # via pytest +pyasn1==0.4.8 # via -r requirements/base.txt, rsa +pymongo==3.11.0 # via -r requirements/base.txt, edx-opaque-keys +pyparsing==2.4.7 # via packaging +pytest-cov==2.10.1 # via -r requirements/test.in +pytest==6.1.2 # via -r requirements/test.in, pytest-cov +python-dateutil==2.8.1 # via -r requirements/base.txt, botocore +python-gnupg==0.4.6 # via -r requirements/base.txt +pyyaml==5.3.1 # via -r requirements/base.txt, awscli +rsa==4.5 # via -r requirements/base.txt, awscli +s3transfer==0.3.3 # via -r requirements/base.txt, awscli +six==1.15.0 # via -r requirements/base.txt, edx-ccx-keys, edx-opaque-keys, packaging, python-dateutil +stevedore==3.2.2 # via -r requirements/base.txt, edx-opaque-keys +toml==0.10.2 # via pytest +urllib3==1.25.11 # via -r requirements/base.txt, botocore +zipp==3.4.0 # via -r requirements/base.txt, importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/tox.in b/requirements/tox.in new file mode 100644 index 0000000..ed6c885 --- /dev/null +++ b/requirements/tox.in @@ -0,0 +1,3 @@ +-c constraints.txt + +tox \ No newline at end of file diff --git a/requirements/tox.txt b/requirements/tox.txt new file mode 100644 index 0000000..0b365b1 --- /dev/null +++ b/requirements/tox.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +appdirs==1.4.4 # via virtualenv +distlib==0.3.1 # via virtualenv +filelock==3.0.12 # via tox, virtualenv +importlib-metadata==2.0.0 # via pluggy, tox, virtualenv +packaging==20.4 # via tox +pluggy==0.13.1 # via tox +py==1.9.0 # via tox +pyparsing==2.4.7 # via packaging +six==1.15.0 # via packaging, tox, virtualenv +toml==0.10.2 # via tox +tox==3.20.1 # via -r requirements/tox.in +virtualenv==20.1.0 # via tox +zipp==3.4.0 # via importlib-metadata diff --git a/test_requirements.txt b/test_requirements.txt deleted file mode 100644 index 47ab897..0000000 --- a/test_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -setuptools<45 -tox diff --git a/tox.ini b/tox.ini index 90ce738..7c7bf2c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py27 +envlist = py37 [testenv] deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/github_requirements.txt + -r{toxinidir}/requirements/test.txt commands = py.test {posargs} From 3282f8ffe5a91769a8b5afe3a58e31c86845b5e7 Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Mon, 4 Jan 2021 22:50:10 -0500 Subject: [PATCH 2/2] Add authenticationDatabase param to the mongoexport command. Making the requirements Python 3.5 compatible. Allow passing empty username to the mongoexport command. Avoid failing on non-string values. --- exporter/mysql_query.py | 2 +- exporter/tasks.py | 3 ++- requirements/base.in | 2 ++ requirements/base.txt | 25 ++++++++++++++----------- requirements/constraints.txt | 8 ++++++++ sample-config.yaml | 11 ++++++++++- 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/exporter/mysql_query.py b/exporter/mysql_query.py index ea06d3f..55e305e 100644 --- a/exporter/mysql_query.py +++ b/exporter/mysql_query.py @@ -66,4 +66,4 @@ def _write_results_to_tsv(self, cursor, output_file): def _normalize_value(self, value): if value is None: value='NULL' - return six.text_type(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') + return six.text_type(value).replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n') diff --git a/exporter/tasks.py b/exporter/tasks.py index 772b7ad..e845bb6 100644 --- a/exporter/tasks.py +++ b/exporter/tasks.py @@ -175,7 +175,8 @@ class MongoTask(Task): mongoexport --host {mongo_host} --db {mongo_db} - --username {mongo_user} + --authenticationDatabase "{mongo_auth_db}" + --username "{mongo_user}" --password "{mongo_password}" --collection {mongo_collection} --query '{query}' diff --git a/requirements/base.in b/requirements/base.in index e819381..77869e8 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -5,6 +5,8 @@ docopt edx-ccx-keys edx-opaque-keys graphitesend +mysql-connector-python +path path.py pbr pip diff --git a/requirements/base.txt b/requirements/base.txt index 41ce6f5..d84ecb8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,31 +4,34 @@ # # make upgrade # -awscli==1.18.174 # via -r requirements/base.in -botocore==1.19.14 # via awscli, s3transfer +awscli==1.18.204 # via -r requirements/base.in +botocore==1.19.44 # via awscli, s3transfer colorama==0.4.3 # via awscli docopt==0.6.2 # via -r requirements/base.in docutils==0.15.2 # via awscli edx-ccx-keys==1.1.0 # via -r requirements/base.in -edx-opaque-keys==2.1.1 # via -r requirements/base.in, edx-ccx-keys +edx-opaque-keys==2.1.0 # via -c requirements/constraints.txt, -r requirements/base.in, edx-ccx-keys graphitesend==0.10.0 # via -r requirements/base.in -importlib-metadata==2.0.0 # via stevedore +importlib-metadata==1.6.0 # via -c requirements/constraints.txt, path jmespath==0.10.0 # via botocore +more-itertools==8.6.0 # via zipp +mysql-connector-python==8.0.22 # via -r requirements/base.in path.py==12.5.0 # via -r requirements/base.in -path==15.0.0 # via path.py +path==13.1.0 # via -c requirements/constraints.txt, -r requirements/base.in, path.py pbr==5.5.1 # via -r requirements/base.in, stevedore -psutil==5.7.3 # via -r requirements/base.in +protobuf==3.14.0 # via mysql-connector-python +psutil==5.8.0 # via -r requirements/base.in pyasn1==0.4.8 # via rsa -pymongo==3.11.0 # via -r requirements/base.in, edx-opaque-keys +pymongo==3.11.2 # via -r requirements/base.in, edx-opaque-keys python-dateutil==2.8.1 # via -r requirements/base.in, botocore python-gnupg==0.4.6 # via -r requirements/base.in pyyaml==5.3.1 # via -r requirements/base.in, awscli rsa==4.5 # via awscli s3transfer==0.3.3 # via awscli -six==1.15.0 # via edx-ccx-keys, edx-opaque-keys, python-dateutil -stevedore==3.2.2 # via edx-opaque-keys -urllib3==1.25.11 # via botocore -zipp==3.4.0 # via importlib-metadata +six==1.15.0 # via edx-ccx-keys, edx-opaque-keys, protobuf, python-dateutil, stevedore +stevedore==1.32.0 # via -c requirements/constraints.txt, edx-opaque-keys +urllib3==1.26.2 # via botocore +zipp==1.0.0 # via -c requirements/constraints.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e69de29..7631f47 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -0,0 +1,8 @@ +# edx-platform Juniper version compatible. +edx-opaque-keys==2.1.0 + +# Python 3.5 compatible. +path==13.1.0 +stevedore==1.32.0 +importlib-metadata==1.6.0 +zipp==1.0.0 \ No newline at end of file diff --git a/sample-config.yaml b/sample-config.yaml index b388566..67e8bac 100644 --- a/sample-config.yaml +++ b/sample-config.yaml @@ -16,8 +16,9 @@ defaults: django_cms_settings: cms-django-settings e.g. cms.envs.devstack_docker django_database: the_django_database mongo_collection: name-of-the-mongo-collection - sql_user: user-name + sql_user: mysql-username sql_db: name-of-the-edxapp-db e.g. edxapp + sql_password: mysql-password secret_key: '' monitor: false graphite_host: localhost @@ -30,8 +31,13 @@ environments: django_config: /prod/django/config mongo_host: 0.0.0.0:9999 mongo_db: the_mongo_db + mongo_user: the_mongo_username + mongo_password: the_mongo_password + mongo_auth_db: authentication-database-name name: prod-mongodb sql_host: prod-sql-host.example.com + lms_config: path-to-the-lms-yml-config-file + studio_config: path-to-the-studio-yml-config-file edge: django_config: /edge/django/config mongo_host: 0.0.0.0:8888 @@ -44,8 +50,11 @@ environments: mongo_db: edxapp mongo_user: edxapp mongo_password: password + mongo_auth_db: authentication-database-name name: local-mongodb This name will be used to create the Mongo DB files. sql_host: edx.devstack.mysql + lms_config: path-to-the-lms-yml-config-file + studio_config: path-to-the-studio-yml-config-file organizations: edX: