From ef079df7dffde39bea257ac0cc9d0daed6c72a15 Mon Sep 17 00:00:00 2001 From: Alexandre Souza Date: Thu, 18 Feb 2016 18:39:26 -0200 Subject: [PATCH 1/7] handle callable objects that are not functions --- proof/analysis.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/proof/analysis.py b/proof/analysis.py index f4ce294..22909e2 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -110,7 +110,13 @@ def _fingerprint(self): hasher.update(history) - source = inspect.getsource(self._func) + # self._func can be a callable object + if hasattr(self._func, '__call__'): + # get the source from the class + if hasattr(self._func, '__class__'): + source = inspect.getsource(self._func.__class__) + else: + source = inspect.getsource(self._func) # In Python 3 inspect.getsource returns unicode data if six.PY3: From 233b060398c418bd4346821cbda00e5823ca037b Mon Sep 17 00:00:00 2001 From: Alexandre Souza Date: Thu, 18 Feb 2016 18:43:55 -0200 Subject: [PATCH 2/7] pep8 improvements --- proof/analysis.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/proof/analysis.py b/proof/analysis.py index 22909e2..35e5657 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -10,6 +10,8 @@ propagated to all dependent analyses. """ +from __future__ import print_function + import bz2 from copy import deepcopy from glob import glob @@ -19,11 +21,12 @@ try: import cPickle as pickle -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover import pickle import six + class Cache(object): """ Utility class for managing cached data. @@ -59,6 +62,7 @@ def set(self, data): f.write(pickle.dumps(self._data)) f.close() + def never_cache(func): """ Decorator to flag that a given analysis function should never be cached. @@ -67,6 +71,7 @@ def never_cache(func): return func + class Analysis(object): """ An Analysis is a function whose source code fingerprint and output can be @@ -90,7 +95,9 @@ def __init__(self, func, cache_dir='.proof', _trace=[]): self._trace = _trace + [self] self._child_analyses = [] - self._cache_path = os.path.join(self._cache_dir, '%s.cache' % self._fingerprint()) + self._cache_path = os.path.join( + self._cache_dir, '%s.cache' % self._fingerprint() + ) self._cache = Cache(self._cache_path) self._registered_cache_paths = [] From 346d6dca51cd81be8412ce76620ca61c773ef2eb Mon Sep 17 00:00:00 2001 From: Alexandre Souza Date: Thu, 18 Feb 2016 19:16:07 -0200 Subject: [PATCH 3/7] refactoring --- proof/analysis.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/proof/analysis.py b/proof/analysis.py index 35e5657..9e482be 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -117,11 +117,8 @@ def _fingerprint(self): hasher.update(history) - # self._func can be a callable object - if hasattr(self._func, '__call__'): - # get the source from the class - if hasattr(self._func, '__class__'): - source = inspect.getsource(self._func.__class__) + if not inspect.isfunction(self._func) and hasattr(self._func, '__class__'): + source = inspect.getsource(self._func.__class__) else: source = inspect.getsource(self._func) From de3af03172ea3e3af826f22e85df9e3cbb96dac3 Mon Sep 17 00:00:00 2001 From: Alexandre <0x41e@protonmail.ch> Date: Sat, 5 Mar 2016 21:38:54 -0300 Subject: [PATCH 4/7] rename param to 'callable' and small refactoring --- proof/analysis.py | 61 ++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/proof/analysis.py b/proof/analysis.py index 9e482be..6ce0fcd 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -17,6 +17,7 @@ from glob import glob import hashlib import inspect +import logging import os try: @@ -27,6 +28,9 @@ import six +logger = logging.getLogger(__name__) + + class Cache(object): """ Utility class for managing cached data. @@ -46,9 +50,8 @@ def get(self): Get cached data from memory or disk. """ if self._data is None: - f = bz2.BZ2File(self._cache_path) - self._data = pickle.loads(f.read()) - f.close() + with bz2.BZ2File(self._cache_path) as f: + self._data = pickle.loads(f.read()) return deepcopy(self._data) @@ -58,9 +61,8 @@ def set(self, data): """ self._data = data - f = bz2.BZ2File(self._cache_path, 'w') - f.write(pickle.dumps(self._data)) - f.close() + with bz2.BZ2File(self._cache_path, 'w') as f: + f.write(pickle.dumps(self._data)) def never_cache(func): @@ -82,15 +84,15 @@ class Analysis(object): If a parent analysis changes then it and all it's children will be refreshed. - :param func: A callable that implements the analysis. Must accept a `data` + :param callable: A callable that implements the analysis. Must accept a `data` argument that is the state inherited from its ancestors analysis. :param cache_dir: Where to stored the cache files for this analysis. :param _trace: The ancestors this analysis, if any. For internal use only. """ - def __init__(self, func, cache_dir='.proof', _trace=[]): - self._name = func.__name__ - self._func = func + def __init__(self, callable, cache_dir='.proof', _trace=[]): + self._name = callable.__name__ + self._callable = callable self._cache_dir = cache_dir self._trace = _trace + [self] self._child_analyses = [] @@ -109,7 +111,7 @@ def _fingerprint(self): """ hasher = hashlib.md5() - history = '\n'.join([analysis._name for analysis in self._trace]) + history = '\n'.join(analysis._name for analysis in self._trace) # In Python 3 function names can be non-ascii identifiers if six.PY3: @@ -117,10 +119,10 @@ def _fingerprint(self): hasher.update(history) - if not inspect.isfunction(self._func) and hasattr(self._func, '__class__'): - source = inspect.getsource(self._func.__class__) + if not inspect.isfunction(self._callable): + source = inspect.getsource(self._callable.__class__) else: - source = inspect.getsource(self._func) + source = inspect.getsource(self._callable) # In Python 3 inspect.getsource returns unicode data if six.PY3: @@ -139,20 +141,16 @@ def _cleanup_cache_files(self): if path not in self._registered_cache_paths: os.remove(path) - def then(self, child_func): + def then(self, child_callable): """ Create a new analysis which will run after this one has completed with access to the data it generated. - :param func: A callable that implements the analysis. Must accept a + :param child_callable: A callable that implements the analysis. Must accept a `data` argument that is the state inherited from its ancestors analysis. """ - analysis = Analysis( - child_func, - cache_dir=self._cache_dir, - _trace=self._trace - ) + analysis = Analysis(child_callable, self._cache_dir, self._trace) self._child_analyses.append(analysis) @@ -184,31 +182,28 @@ def run(self, refresh=False, _parent_cache=None): if not os.path.exists(self._cache_dir): os.makedirs(self._cache_dir) - do_not_cache = getattr(self._func, 'never_cache', False) + do_not_cache = getattr(self._callable, 'never_cache', False) + + if refresh: + logger.info('Refreshing: {}'.format(self._name)) - if refresh is True: - print('Refreshing: %s' % self._name) elif do_not_cache: refresh = True - - print('Never cached: %s' % self._name) + logger.info('Never cached: {}'.format(self._name)) elif not self._cache.check(): refresh = True - print('Stale cache: %s' % self._name) + logger.info('Stale cache: {}'.format(self._name)) if refresh: - if _parent_cache: - local_data = _parent_cache.get() - else: - local_data = {} + local_data = _parent_cache.get() if _parent_cache else {} - self._func(local_data) + self._callable(local_data) if not do_not_cache: self._cache.set(local_data) else: - print('Deferring to cache: %s' % self._name) + logger.info('Deferring to cache: {}'.format(self._name)) for analysis in self._child_analyses: analysis.run(refresh=refresh, _parent_cache=self._cache) From 9dfb3e0a3759da64926a2285bb08a3249780a6ca Mon Sep 17 00:00:00 2001 From: Alexandre <0x41e@protonmail.ch> Date: Sat, 5 Mar 2016 21:42:30 -0300 Subject: [PATCH 5/7] avoid confusion with built-in function --- proof/analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proof/analysis.py b/proof/analysis.py index 6ce0fcd..ddce2aa 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -90,9 +90,9 @@ class Analysis(object): :param _trace: The ancestors this analysis, if any. For internal use only. """ - def __init__(self, callable, cache_dir='.proof', _trace=[]): - self._name = callable.__name__ - self._callable = callable + def __init__(self, _callable, cache_dir='.proof', _trace=[]): + self._name = _callable.__name__ + self._callable = _callable self._cache_dir = cache_dir self._trace = _trace + [self] self._child_analyses = [] From 47e001b51776893a30ca540c2e62f4d1e1cfda54 Mon Sep 17 00:00:00 2001 From: Alexandre <0x41e@protonmail.ch> Date: Sat, 5 Mar 2016 22:23:19 -0300 Subject: [PATCH 6/7] skip tearDown error --- tests/test_proof.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_proof.py b/tests/test_proof.py index acf6619..5dff43d 100644 --- a/tests/test_proof.py +++ b/tests/test_proof.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from copy import deepcopy from glob import glob @@ -30,7 +30,7 @@ def setUp(self): self.executed_stage_never_cache = 0 def tearDown(self): - shutil.rmtree(TEST_CACHE) + shutil.rmtree(TEST_CACHE, ignore_errors=True) def stage1(self, data): self.executed_stage1 += 1 From 507234d851b75f3dc19990dd3de1039e0bd26ca7 Mon Sep 17 00:00:00 2001 From: Alexandre <0x41e@protonmail.ch> Date: Sat, 5 Mar 2016 22:26:52 -0300 Subject: [PATCH 7/7] handling functions or methods in the same way --- proof/analysis.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/proof/analysis.py b/proof/analysis.py index ddce2aa..e1db426 100644 --- a/proof/analysis.py +++ b/proof/analysis.py @@ -84,7 +84,7 @@ class Analysis(object): If a parent analysis changes then it and all it's children will be refreshed. - :param callable: A callable that implements the analysis. Must accept a `data` + :param _callable: A callable that implements the analysis. Must accept a `data` argument that is the state inherited from its ancestors analysis. :param cache_dir: Where to stored the cache files for this analysis. :param _trace: The ancestors this analysis, if any. For internal use @@ -119,10 +119,14 @@ def _fingerprint(self): hasher.update(history) - if not inspect.isfunction(self._callable): - source = inspect.getsource(self._callable.__class__) - else: + function_or_method = ( + inspect.isfunction(self._callable), inspect.ismethod(self._callable) + ) + + if any(function_or_method): source = inspect.getsource(self._callable) + else: + source = inspect.getsource(self._callable.__class__) # In Python 3 inspect.getsource returns unicode data if six.PY3: