Add support for python 3.7 and drop python 2 support

.travis.yml

@@ -1,10 +1,8 @@
 # Config file for automatic testing at travis-ci.org
 
 language: python
-matrix:
-  include:
-    - python: 2.7
-      env: TOXENV=py27
+python:
+  - 3.7
 
 cache:
   - pip
@@ -13,7 +11,7 @@ before_install:
   - pip install --upgrade pip
 
 install:
-  - pip install -r test_requirements.txt
+  - pip install -r requirements/tox.txt
 
 script:
   - tox

AUTHORS

@@ -1,12 +1,21 @@
-Carlos Andrés Rocha <[email protected]>
-John Jarvis <[email protected]>
-Isaac Chuang <[email protected]>
-Ed Zarecor  <[email protected]>
+Aarif <[email protected]>
+Abdul Mannan <[email protected]>
+Alex Dusenbery <[email protected]>
+Alex Dusenbery <[email protected]>
+Andrew Zafft <[email protected]>
+Brian Beggs <[email protected]>
+Brian Beggs <[email protected]>
 Brian Wilson <[email protected]>
-Gabe Mulley <[email protected]>
-Greg Price <[email protected]>
+Dennis Jen <[email protected]>
+Diana Huang <[email protected]>
 Diana Huang <[email protected]>
-Andy Armstrong <[email protected]>
-Will Daly <[email protected]>
-Eric Fischer <[email protected]>
-Dennis Jen <[email protected]>
+Feanil Patel <[email protected]>
+Gabe Mulley <[email protected]>
+Gabe Mulley <[email protected]>
+Gregory Martin <[email protected]>
+Gregory Martin <[email protected]>
+Hassan <[email protected]>
+Hassan Javeed <[email protected]>
+Stu Young <[email protected]>
+Victor Shnayder <[email protected]>
+brianhw <[email protected]>

Makefile

@@ -3,4 +3,14 @@ clean:
 	rm -rf exporter/tests/__pycache__/
 
 test:
-	py.test
+	pytest
+
+
+upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	pip install -qr requirements/pip-tools.txt
+	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip-compile --upgrade -o requirements/base.txt requirements/base.in
+	pip-compile --upgrade -o requirements/test.txt requirements/test.in
+	pip-compile --upgrade -o requirements/github_requirements.txt requirements/github.in
+	pip-compile --upgrade -o requirements/tox.txt requirements/tox.in

exporter/config.py

@@ -10,6 +10,7 @@
 import yaml
 
 from exporter.util import merge, filter_keys
+import six
 
 
 WORK_SUBDIR = 'course-data'
@@ -59,7 +60,7 @@ def update_config(config, program_options):
 def merge_program_options(config, program_options):
     # get program options, removing '--' and replacing '-' with '_'
     options = {k[2:].replace('-', '_'): v for k, v
-               in program_options.iteritems()
+               in six.iteritems(program_options)
                if k.startswith('--')}
 
     config['options'] = options
@@ -104,7 +105,7 @@ def update_environments(config):
     for env in ['prod', 'edge']:
         if env in environments:
             data = environments.get(env, {})
-            for config_name, token_name in field_map.iteritems():
+            for config_name, token_name in six.iteritems(field_map):
                 data[config_name] = tokens.get(token_name)
 
             # different settings for edge
@@ -123,7 +124,7 @@ def update_organizations(config):
 
     # lowercase orgs before selection
     organizations = {org.lower(): values for org, values
-                     in config['organizations'].iteritems()}
+                     in six.iteritems(config['organizations'])}
 
     # select only organizations in arguments
     organizations = filter_keys(organizations, values.get('org'))

exporter/course_export.py

@@ -49,6 +49,7 @@
 from exporter.main import run_tasks, archive_directory, upload_data, get_all_courses, _get_selected_tasks
 from exporter.config import setup, get_config_for_env, get_config_for_course
 from exporter.util import make_temp_directory, with_temp_directory, merge
+import six
 
 log = logging.getLogger(__name__)
 
@@ -150,16 +151,12 @@ def get_filename_safe_course_id(course_id, replacement_char='_'):
     """
     try:
         course_key = CourseKey.from_string(course_id)
-        filename = unicode(replacement_char).join([course_key.org, course_key.course, course_key.run])
+        filename = six.text_type(replacement_char).join([course_key.org, course_key.course, course_key.run])
     except InvalidKeyError:
         # If the course_id doesn't parse, we will still return a value here.
         filename = course_id
 
     # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
     # We represent the first four with \w.
     # TODO: Once we support courses with unicode characters, we will need to revisit this.
-    return re.sub(r'[^\w\.\-]', unicode(replacement_char), filename)
-
-if __name__ == '__main__':
-    import sys
-    sys.exit(main())
+    return re.sub(r'[^\w\.\-]', six.text_type(replacement_char), filename)

exporter/main.py

@@ -63,11 +63,10 @@
 from exporter.util import make_temp_directory, with_temp_directory
 from exporter.util import filter_keys, memoize, execute_shell
 from exporter.util import logging_streams_on_failure
-
+import six
 
 log = logging.getLogger(__name__)
 
-
 # pylint: disable=missing-docstring
 
 
@@ -77,7 +76,6 @@
 def main(argv=None):
     general_config = setup(__doc__, argv=argv)
     for organization in general_config['organizations']:
-
         config = get_config_for_org(general_config, organization)
 
         with make_org_directory(config, organization) as destination:
@@ -304,12 +302,13 @@ def match(course):
 
     return [course for course in courses if match(course)]
 
+
 def get_all_courses(**kwargs):
     log.info('Retrieving all courses')
 
     # make a set of fixed arguments, so we can memoize
     kwargs = {
-        k: v for k, v in kwargs.iteritems()
+        k: v for k, v in six.iteritems(kwargs)
         if k.startswith('django') or k == 'lms_config' or k == 'studio_config'
     }
     kwargs['dry_run'] = False  # always query for course names

exporter/mysql_query.py

@@ -2,6 +2,7 @@
 import json
 import csv
 import mysql.connector
+import six
 
 MAX_FETCH_SIZE = 10000
 
@@ -65,4 +66,4 @@ def _write_results_to_tsv(self, cursor, output_file):
 
     def _normalize_value(self, value):
         if value is None: value='NULL'
-        return unicode(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n')
+        return six.text_type(value).encode('utf-8').replace('\\', '\\\\').replace('\r', '\\r').replace('\t','\\t').replace('\n', '\\n')

exporter/properties.py

@@ -55,7 +55,7 @@ def export_properties(config, directory, files=None, orgs=None, prefix=''):
     recreate_directory(directory)
 
     orgs = [o.lower() for o in orgs.split()] if orgs else ['*']
-    print orgs
+    print(orgs)
 
     files_data = load_files(files)