Merge branch 'feature/improved-logging' into feature/codemirror-impro…

…vements
pebble · May 31, 2016 · e746af9 · e746af9
2 parents c35312d + 6009714
commit e746af9
Show file tree

Hide file tree

Showing 18 changed files with 429 additions and 119 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,13 @@
+language: python
+python: "2.7"
+sudo: false
+services:
+  - rabbitmq
+  - postgresql
+before_script:
+  - psql -c "CREATE DATABASE travisci;" -U postgres
+install:
+  - pip install -r requirements.txt
+script: python manage.py test
+env:
+  - AWS_ENABLED=yes
diff --git a/cloudpebble/settings.py b/cloudpebble/settings.py
@@ -1,12 +1,16 @@
 # encoding: utf-8
 # Django settings for cloudpebble project.
 
+import sys
 import os
 import socket
 import dj_database_url
+
 _environ = os.environ
 
 DEBUG = _environ.get('DEBUG', '') != ''
+VERBOSE = DEBUG or (_environ.get('VERBOSE', '') != '')
+TESTING = 'test' in sys.argv
 TEMPLATE_DEBUG = DEBUG
 
 ADMINS = (
@@ -19,7 +23,18 @@
 
 MANAGERS = ADMINS
 
-if 'DATABASE_URL' not in _environ:
+if 'TRAVIS' in _environ:
+    DATABASES = {
+        'default': {
+            'ENGINE':   'django.db.backends.postgresql_psycopg2',
+            'NAME':     'travisci',
+            'USER':     'postgres',
+            'PASSWORD': '',
+            'HOST':     'localhost',
+            'PORT':     '',
+        }
+    }
+elif 'DATABASE_URL' not in _environ:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.postgresql_psycopg2',
@@ -231,32 +246,42 @@
     'djangobower',
 )
 
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error when DEBUG=False.
-# See http://docs.djangoproject.com/en/dev/topics/logging for
-# more details on how to customize your logging configuration.
+# This logging config prints:
+# INFO logs from django
+# INFO or DEBUG logs from 'ide', depending on whether DEBUG=True
+# all WARNING logs from any sources
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
+    'formatters': {
+        'verbose': {
+            'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
+            'datefmt': "%d/%b/%Y %H:%M:%S"
+        },
     },
     'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
         }
     },
     'loggers': {
-        'django.request': {
-            'handlers': ['mail_admins'],
-            'level': 'ERROR',
-            'propagate': True,
+        'django': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': True
         },
+        'ide': {
+            'handlers': ['console'],
+            'level': 'DEBUG' if VERBOSE else 'INFO',
+            'propagate': True
+        },
+        '': {
+            'handlers': ['console'],
+            'level': 'WARNING',
+            'propagate': False
+        }
     }
 }
 
@@ -309,6 +334,7 @@
 AWS_S3_SOURCE_BUCKET = _environ.get('AWS_S3_SOURCE_BUCKET', 'source.cloudpebble.net')
 AWS_S3_BUILDS_BUCKET = _environ.get('AWS_S3_BUILDS_BUCKET', 'builds.cloudpebble.net')
 AWS_S3_EXPORT_BUCKET = _environ.get('AWS_S3_EXPORT_BUCKET', 'export.cloudpebble.net')
+AWS_S3_HOST = _environ.get('AWS_S3_HOST', None)
 AWS_S3_FAKE_S3 = _environ.get('AWS_S3_FAKE_S3', None)
 
 TYPOGRAPHY_CSS = _environ.get('TYPOGRAPHY_CSS', None)

diff --git a/ide/api/qemu.py b/ide/api/qemu.py
@@ -1,19 +1,22 @@
 __author__ = 'katharine'
 
 import json
+import requests
+import random
+import urlparse
+import string
+import logging
+
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
-from django.http import HttpResponseNotFound
 from django.shortcuts import redirect, render
 from django.views.decorators.http import require_POST
+
 from ide.api import json_response, json_failure
-import requests
-import random
-import urlparse
-import urllib
-import string
 from utils.redis_helper import redis_client
 
+logger = logging.getLogger(__name__)
+
 
 @login_required
 @require_POST
@@ -38,14 +41,12 @@ def launch_emulator(request):
             response.raise_for_status()
             response = response.json()
         except (requests.RequestException, ValueError) as e:
-            print "couldn't fetch old instance: %s" % e
-            pass
+            logger.info("couldn't fetch old instance: %s", e)
         else:
             if response.get('alive', False):
                 return json_response(qemu_instance)
             else:
-                print "old instance is dead."
-
+                logger.info("old instance is dead.")
 
     token = _generate_token()
     servers = set(settings.QEMU_URLS)
@@ -74,10 +75,9 @@ def launch_emulator(request):
             redis_client.set(redis_key, json.dumps(response))
             return json_response(response)
         except requests.HTTPError as e:
-            print e.response.text
+            logger.warning("Got HTTP error from QEMU launch. Content:\n%s", e.response.text)
         except (requests.RequestException, ValueError) as e:
-            print e
-            pass
+            logger.error("Error launching qemu: %s", e)
     return json_failure("Unable to create emulator instance.")
 
 

diff --git a/ide/api/ycm.py b/ide/api/ycm.py
@@ -1,4 +1,8 @@
 import json
+import logging
+import random
+
+import requests
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.shortcuts import get_object_or_404
@@ -8,12 +12,11 @@
 
 from ide.api import json_response, json_failure
 from ide.models.project import Project
-import requests
-import random
-
 
 __author__ = 'katharine'
 
+logger = logging.getLogger(__name__)
+
 
 @login_required
 @require_POST
@@ -32,7 +35,7 @@ def init_autocomplete(request, project_id):
             if f.kind == 'png-trans':
                 resource_ids.extend([
                     '#define RESOURCE_ID_%s_BLACK %d' % (identifier.resource_id, count),
-                    '#define RESOURCE_ID_%s_WHITE %d' % (identifier.resource_id, count+1)
+                    '#define RESOURCE_ID_%s_WHITE %d' % (identifier.resource_id, count + 1)
                 ])
                 count += 2
             else:
@@ -71,6 +74,6 @@ def _spin_up_server(request):
         except (requests.RequestException, ValueError):
             import traceback
             traceback.print_exc()
-        print "Server %s failed; trying another." % server
+        logger.warning("Server %s failed; trying another.", server)
     # If we get out of here, something went wrong.
     return json_failure({'success': False, 'error': 'No servers'})
diff --git a/ide/git.py b/ide/git.py
@@ -1,13 +1,17 @@
-from ide.models.user import UserGithub
-from django.utils.translation import ugettext as _
-
-from github import Github, BadCredentialsException, UnknownObjectException
-from github.NamedUser import NamedUser
-from django.conf import settings
 import base64
 import json
 import urllib2
 import re
+import logging
+
+from github import Github, BadCredentialsException, UnknownObjectException
+from github.NamedUser import NamedUser
+from django.utils.translation import ugettext as _
+from django.conf import settings
+
+from ide.models.user import UserGithub
+
+logger = logging.getLogger(__name__)
 
 
 def git_auth_check(f):
@@ -19,12 +23,13 @@ def g(user, *args, **kwargs):
         except BadCredentialsException:
             # Bad credentials; remove the user's auth data.
             try:
-                print "Bad credentials; revoking user's github tokens."
+                logger.warning("Bad credentials; revoking user's github tokens.")
                 github = user.github
                 github.delete()
             except:
                 pass
             raise
+
     return g
 
 

diff --git a/ide/models/files.py b/ide/models/files.py
@@ -3,20 +3,23 @@
 import traceback
 import datetime
 import json
+import logging
+
 from django.conf import settings
-from django.core.validators import RegexValidator
 from django.db import models
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
 from django.utils.timezone import now
 from django.core.validators import RegexValidator
 from django.utils.translation import ugettext as _
-import utils.s3 as s3
 
+import utils.s3 as s3
 from ide.models.meta import IdeModel
 
 __author__ = 'katharine'
 
+logger = logging.getLogger(__name__)
+
 
 class ResourceFile(IdeModel):
     project = models.ForeignKey('Project', related_name='resources')
@@ -355,7 +358,7 @@ def delete_file(sender, instance, **kwargs):
             try:
                 s3.delete_file('source', instance.s3_path)
             except:
-                traceback.print_exc()
+                logger.exception("Failed to delete S3 file")
         else:
             try:
                 os.unlink(instance.local_filename)

diff --git a/ide/tasks/archive.py b/ide/tasks/archive.py
@@ -5,6 +5,8 @@
 import uuid
 import zipfile
 import json
+import logging
+
 from celery import task
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -20,6 +22,8 @@
 
 __author__ = 'katharine'
 
+logger = logging.getLogger(__name__)
+
 
 def add_project_to_archive(z, project, prefix=''):
     source_files = SourceFile.objects.filter(project=project)
@@ -221,7 +225,7 @@ def make_valid_filename(zip_entry):
                             raise ValueError("Generic resource filenames cannot contain a tilde (~)")
                         if file_name not in desired_resources:
                             desired_resources[root_file_name] = []
-                        print "Desired resource: %s" % root_file_name
+
                         desired_resources[root_file_name].append(resource)
                         file_exists_for_root[root_file_name] = False
 
@@ -233,18 +237,18 @@ def make_valid_filename(zip_entry):
                             try:
                                 extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, base_filename))
                             except KeyError:
-                                print "Failed to open %s" % base_filename
+                                logger.debug("Failed to open %s", base_filename)
                                 continue
 
                             # Now we know the file exists and is in the resource directory - is it the one we want?
                             tags, root_file_name = get_filename_variant(base_filename, tag_map)
                             tags_string = ",".join(str(int(t)) for t in tags)
 
-                            print "Importing file %s with root %s " % (entry.filename, root_file_name)
+                            logger.debug("Importing file %s with root %s ", entry.filename, root_file_name)
 
                             if root_file_name in desired_resources:
                                 medias = desired_resources[root_file_name]
-                                print "Looking for variants of %s" % root_file_name
+                                logger.debug("Looking for variants of %s", root_file_name)
 
                                 # Because 'kind' and 'is_menu_icons' are properties of ResourceFile in the database,
                                 # we just use the first one.
@@ -260,7 +264,7 @@ def make_valid_filename(zip_entry):
                                         is_menu_icon=is_menu_icon)
 
                                 # But add a resource variant for every file
-                                print "Adding variant %s with tags [%s]" % (root_file_name, tags_string)
+                                logger.debug("Adding variant %s with tags [%s]", root_file_name, tags_string)
                                 actual_file_name = resource['file']
                                 resource_variants[actual_file_name] = ResourceVariant.objects.create(resource_file=resources_files[root_file_name], tags=tags_string)
                                 resource_variants[actual_file_name].save_file(extracted)