diff --git a/.travis.yml b/.travis.yml index b7cf3de..7980b1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,20 @@ language: python +sudo: required +dist: xenial python: - 3.5 - 3.6 -script: make test + - 3.7 +script: make test-coverage notifications: email: false -install: +services: + - postgresql +before_script: - psql -c 'CREATE DATABASE pgcrypto_fields' -U postgres - - pip install -r requirements.txt +install: + - pip install -e . + - pip install -r requirements_dev.txt - pip install $DJANGO env: matrix: diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..8ba6b31 --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,13 @@ +# Credits + + +## Development Lead + +* Charlie Denton +* Kévin Etienne +* Peter J. Farrell +* Max Peterson + +## Contributors + +None yet. Why not be the first? diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab0328..98b9b18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,17 @@ -### 2.3.0 +## 2.4.0 + +* Added auto-decryption of all encrypted fields including FK tables +* Removed django-pgcrypto-fields `aggregates`, `PGPManager` and `PGPAdmin` as they are no longer needed +* Added support for `get_or_create()` and `update_or_create()` (#27) +* Added support for `get_by_natural_key()` (#23) +* Added support for `only()` and `defer()` as they were not supported with `PGPManager` +* Added support for `distinct()` (Django 2.1+ with workaround available for 2.0 and lower) +* Separated out dev requirements from setup.py requirements +* Updated packaging / setup.py to include long description +* Added AUTHORS and updated CONTRIBUTING +* Updated TravisCI to use Xenial to gain Python 3.7 in the matrix + +## 2.3.1 * Added `__range` lookup for Date / DateTime fields (#59) * Remove compatibility for `Django 1.8, 1.9, and 1.10` (#62) @@ -10,6 +23,10 @@ * Updated Travis config to include Python 3.5 and 3.6 * Refactored lookups and mixins +## 2.3.0 + +* Invalid release, bump to 2.3.1 + ## 2.2.0 * Merge `.coveragerc` into `setup.cfg` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bb7ec1..655f13e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,25 +1,117 @@ # Contributing to Django-PGCrypto-Fields -We welcome contributions in many forms: +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: * Code patches and enhancements * Documentation improvements * Bug reports and patch reviews -## Running Tests +## Types of Contributions + +### Report Bugs + +Report bugs at https://github.com/incuna/django-pgcrypto-fields/issues + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +### Fix Bugs + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +### Implement Features + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +### Write Documentation + +django-pgcrypto-fields could always use more documentation, whether as part of the +official django-pgcrypto-fields docs, in docstrings, or even on the web in blog posts, +articles, and such. + +### Submit Feedback + +The best way to send feedback is to file an issue at https://github.com/incuna/django-pgcrypto-fields/issues + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :-) + +## Get Started! + +Ready to contribute? Here's how to set up `django-pgcrypto-fields` for local development. + +1. Fork the `django-pgcrypto-fields` repo on GitHub. +2. Clone your fork locally: + + ```bash + $ git clone git@github.com:your_name_here/pgcrypto.git + ``` + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development: + + ```bash + $ mkvirtualenv django-pgcrypto-fields + $ cd django-pgcrypto-fields/ + $ pip install -r requirements_dev.txt --upgrade + ``` + +4. Create a branch for local development: + + ```bash + $ git checkout -b name-of-your-bugfix-or-feature + ``` + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the + tests: + + ```bash + $ make test + ``` + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub: + + ```bash + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + ``` + +7. Submit a pull request through the GitHub website. -* Install requirements to a virtual environment -* Setup a local PostgreSQL server -* Create a PostreSQL database named `pgcrypto_fields` -* In a terminal, run `make test` +### Pull Request Guidelines +Before you submit a pull request, check that it meets these guidelines: -## Releasing to PyPI +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.4, 3.5 and 3.7. Check + https://travis-ci.org/incuna/django-pgcrypto-fields/pull_requests + and make sure that the tests pass for all supported Python versions. -This section only applies to maintainers. +### Deploying -In your virtual environment, run +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in CHANGELOG.md). +Then run: -* `pip install pip --upgrade` -* `pip install setuptools wheel twine` -* `make release` +```bash +$ make release +``` diff --git a/Makefile b/Makefile index 56e4b3a..709a9fa 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,55 @@ SHELL := /bin/bash +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + help: - @echo "Usage:" - @echo " make release | Release to pypi." - @echo " make test | Run the tests." + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -release: +lint: ## Check style with flake8 + @flake8 . + +clean-build: ## Remove build artifacts + rm -r -f dist/* + rm -r -f build/* + rm -fr htmlcov/ + +build: clean-build ## Builds source and wheel package python setup.py sdist bdist_wheel + ls -l dist + +release: ## Package and upload a release twine upload dist/* -test: - @coverage run ./tests/run.py - @coverage report - @flake8 . +test: clean-build lint ## Run tests quickly with the default Python + ./tests/run.py + +test-coverage: ## Check code coverage quickly with the default Python + coverage run ./tests/run.py + coverage report -m + +test-coverage-html: test-coverage ## Check code coverage quickly with the default Python and show report + coverage html + $(BROWSER) htmlcov/index.html diff --git a/README.md b/README.md index 780e05b..0a37d86 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# django-pgcrypto-fields [![Latest Release](https://img.shields.io/pypi/v/django-pgcrypto-fields.svg)](https://pypi.org/pypi/django-pgcrypto-fields/) [![Python Versions](https://img.shields.io/pypi/pyversions/django-pgcrypto-fields.svg)](https://pypi.org/pypi/django-pgcrypto-fields/) [![Build Status](https://travis-ci.org/incuna/django-pgcrypto-fields.svg?branch=master)](https://travis-ci.org/incuna/django-pgcrypto-fields?branch=master) [![Requirements Status](https://requires.io/github/incuna/django-pgcrypto-fields/requirements.svg?branch=master)](https://requires.io/github/incuna/django-pgcrypto-fields/requirements/?branch=master) +# django-pgcrypto-fields [![Latest Release](https://img.shields.io/pypi/v/django-pgcrypto-fields.svg)](https://pypi.org/pypi/django-pgcrypto-fields/) [![Python Versions](https://img.shields.io/pypi/pyversions/django-pgcrypto-fields.svg)](https://pypi.org/pypi/django-pgcrypto-fields/) [![Build Status](https://travis-ci.org/incuna/django-pgcrypto-fields.svg?branch=master)](https://travis-ci.org/incuna/django-pgcrypto-fields?branch=master) [![Requirements Status](https://requires.io/github/incuna/django-pgcrypto-fields/requirements.svg?branch=master)](https://requires.io/github/incuna/django-pgcrypto-fields/requirements/?branch=master) [![PyUp - Python 3](https://pyup.io/repos/github/incuna/django-pgcrypto-fields/python-3-shield.svg)](https://pyup.io/repos/github/incuna/django-pgcrypto-fields/) + `django-pgcrypto-fields` is a `Django` extension which relies upon `pgcrypto` to encrypt and decrypt data for fields. @@ -28,6 +29,21 @@ INSTALLED_APPS = ( ) ``` +## Upgrading to 2.4.0 from previous versions + +The 2.4.0 version of this library received a large rewrite in order to support +auto-decryption when getting encrypted field data as well as the ability to filter +on encrypted fields without using the old PGPCrypto aggregate functions available +in previous versions. + +The following items in this library have been removed and therefore references in +your application to these items need to be removed as well: + +* `managers.PGPManager` +* `admin.PGPAdmin` +* `aggregates.*` + + ## Fields `django-pgcrypto-fields` has 3 kinds of fields: @@ -121,6 +137,7 @@ N.B. `DatePGPSymmetricKeyField` and `DateTimePGPSymmetricKeyField` only support In `settings.py`: ```python +import os BASEDIR = os.path.dirname(os.path.dirname(__file__)) PUBLIC_PGP_KEY_PATH = os.path.abspath(os.path.join(BASEDIR, 'public.key')) PRIVATE_PGP_KEY_PATH = os.path.abspath(os.path.join(BASEDIR, 'private.key')) @@ -137,25 +154,20 @@ PGCRYPTO_KEY='ultrasecret' # And add 'pgcrypto' to `INSTALLED_APPS` to create the extension for # pgcrypto (in a migration). INSTALLED_APPS = ( - ... 'pgcrypto', - ... + # Other installed apps ) ``` ### Usage -#### Model / Manager definition +#### Model Definition ```python from django.db import models -from pgcrypto import fields, managers - -class MyModelManager(managers.PGPManager): - pass - +from pgcrypto import fields class MyModel(models.Model): digest_field = fields.TextDigestField() @@ -174,16 +186,14 @@ class MyModel(models.Model): pgp_sym_field = fields.TextPGPSymmetricKeyField() date_pgp_sym_field = fields.DatePGPSymmetricKeyField() datetime_pgp_sym_field = fields.DateTimePGPSymmetricKeyField() - - objects = MyModelManager() ``` #### Encrypting -Data is encrypted when inserted into the database. +Data is automatically encrypted when inserted into the database. Example: -```python +``` >>> MyModel.objects.create(value='Value to be encrypted...') ``` @@ -191,8 +201,11 @@ Hash fields can have hashes auto updated if you use the `original` attribute. Th attribute allows you to indicate another field name to base the hash value on. ```python +from django.db import models -class User(models.Models): +from pgcrypto import fields + +class User(models.Model): first_name = fields.TextPGPSymmetricKeyField(max_length=20, verbose_name='First Name') first_name_hashed = fields.TextHMACField(original='first_name') ``` @@ -202,71 +215,48 @@ take the unencrypted value from the first_name model field as the input value to create the hash. If you did not specify an original attribute, the field would work as it does now and would remain backwards compatible. -#### Decryption using custom model managers - -If you use the bundled `PGPManager` with your custom model manager, all encrypted -fields will automatically decrypted for you (except for hash fields which are one -way). - -N.B. The bundled manager does not support decryption of fields from FK joins. For -example if the `MyModel` class had a FK to to `AnotherModel` class, no encrypted -fields be decrypted in the joined `AnotherModel`. - -It is recommended that you use the bundled `PGPAdmin` class if using the custom -model manager and the Django Admin. The Django Admin performance suffers when -using the bundled custom manager. The `PGPAdmin` disables automatic decryption -for all ORM calls for that admin class. - -```python -from django.contrib import admin - -from pgcrypto.admin import PGPAdmin - - -class MyModelAdmin(admin.ModelAdmin, PGPAdmin): - # Your admin code here -``` - -#### Decrypting using aggregates - -This is useful if you are not using the custom manager or need to decrypt fields -coming from joined FK fields. - ##### PGP fields When accessing the field name attribute on a model instance we are getting the decrypted value. Example: -```python +``` >>> # When using a PGP public key based encryption >>> my_model = MyModel.objects.get() ->>> my_model.value # field's proxy +>>> my_model.value 'Value decrypted' ``` -To be able to filter PGP values we first need to use an aggregate method to -decrypt the values. +Filtering encrypted values is now handled automatically as of 2.4.0. And `aggregate` +methods are not longer supported and have been removed from the library. + +Also, auto-decryption is support for `select_related()` models. -Example when using a `PGPPublicKeyField`: ```python ->>> from pgcrypto.aggregates import PGPPublicKeyAggregate ->>> my_models = MyModel.objects.annotate(PGPPublicKeyAggregate('pgp_pub_field')) -[, ] ->>> my_models.filter(pgp_pub_field__decrypted='Value decrypted') -[] ->>> my_models.first().pgp_pub_field__decrypted -'Value decrypted' +from django.db import models + +from pgcrypto import fields + + +class EncryptedFKModel(models.Model): + fk_pgp_sym_field = fields.TextPGPSymmetricKeyField(blank=True, null=True) + + +class EncryptedModel(models.Model): + pgp_sym_field = fields.TextPGPSymmetricKeyField(blank=True, null=True) + fk_model = models.ForeignKey( + EncryptedFKModel, blank=True, null=True, on_delete=models.CASCADE + ) ``` -Example when using a `PGPSymmetricKeyField`: -```python ->>> from pgcrypto.aggregates import PGPSymmetricKeyAggregate ->>> my_models = MyModel.objects.annotate(PGPSymmetricKeyAggregate('pgp_sym_field')) -[, ] ->>> my_models.filter(pgp_pub_field__decrypted='Value decrypted') -[] ->>> my_models.first().pgp_sym_field__decrypted +Example: +``` +>>> import EncryptedModel +>>> my_model = EncryptedModel.objects.get().select_releated('fk_model') +>>> my_model.pgp_sym_field +'Value decrypted' +>>> my_model.fk_model.fk_pgp_sym_field 'Value decrypted' ``` @@ -276,7 +266,7 @@ To filter hash based values we need to compare hashes. This is achieved by using a `__hash_of` lookup. Example: -```python +``` >>> my_model = MyModel.objects.filter(digest_field__hash_of='value') [] >>> my_model = MyModel.objects.filter(hmac_field__hash_of='value') @@ -284,6 +274,45 @@ Example: ``` +## Limitations + +#### `.distinct('encrypted_field_name')` + +Due to a missing feature in the Django ORM, using `distinct()` on an encrypted field +does not work for Django 2.0.x and lower. + +The normal distinct works on Django 2.1.x and higher: + +```python +items = EncryptedFKModel.objects.filter( + pgp_sym_field__startswith='P' +).only( + 'id', 'pgp_sym_field', 'fk_model__fk_pgp_sym_field' +).distinct( + 'pgp_sym_field' +) +``` + +Workaround for Django 2.0.x and lower: + +```python +from django.db import models + +items = EncryptedFKModel.objects.filter( + pgp_sym_field__startswith='P' +).annotate( + _distinct=models.F('pgp_sym_field') +).only( + 'id', 'pgp_sym_field', 'fk_model__fk_pgp_sym_field' +).distinct( + '_distinct' +) +``` + +This works because the annotated field is auto-decrypted by Django as a `F` field and that +field is used in the `distinct()`. + + ## Security Limitations Taken direction from the PostgreSQL documentation: diff --git a/pgcrypto/__init__.py b/pgcrypto/__init__.py index 2b2edfc..8c45161 100644 --- a/pgcrypto/__init__.py +++ b/pgcrypto/__init__.py @@ -17,8 +17,8 @@ PGP_PUB_ENCRYPT_SQL = "pgp_pub_encrypt(%s, dearmor('{}'))".format( settings.PUBLIC_PGP_KEY, ) -PGP_PUB_DECRYPT_SQL = "pgp_pub_decrypt(%s, dearmor('{}'))".format( +PGP_PUB_DECRYPT_SQL = "pgp_pub_decrypt(%s, dearmor('{}'))::%s".format( settings.PRIVATE_PGP_KEY, ) PGP_SYM_ENCRYPT_SQL = "pgp_sym_encrypt(%s, '{}')".format(settings.PGCRYPTO_KEY) -PGP_SYM_DECRYPT_SQL = "pgp_sym_decrypt(%s, '{}')".format(settings.PGCRYPTO_KEY) +PGP_SYM_DECRYPT_SQL = "pgp_sym_decrypt(%s, '{}')::%s".format(settings.PGCRYPTO_KEY) diff --git a/pgcrypto/admin.py b/pgcrypto/admin.py deleted file mode 100644 index 5284a08..0000000 --- a/pgcrypto/admin.py +++ /dev/null @@ -1,5 +0,0 @@ -class PGPAdmin(object): # pragma: no cover - - def get_queryset(self, request): - """Skip any auto decryption when ORM calls are from the admin.""" - return self.model.objects.get_queryset(**{'skip_decrypt': True}) diff --git a/pgcrypto/aggregates.py b/pgcrypto/aggregates.py deleted file mode 100644 index 7e2ad5d..0000000 --- a/pgcrypto/aggregates.py +++ /dev/null @@ -1,120 +0,0 @@ -from django.conf import settings -from django.db.models import Aggregate - - -class PGPPublicKeySQL: - """Custom SQL aggregate to decrypt a field with public key. - - `PGPPublicKeySQL` provides a SQL template using pgcrypto to decrypt - data from a field in the database. - - `function` defines `pgp_pub_decrypt` which is a pgcrypto SQL function. - This function takes two arguments: - - a encrypted message (bytea); - - a key (bytea). - - `%(function)s` in `template` is populated by `sql_function`. - - `%(field)s` is replaced with the field's name. - - `dearmor` is used to unwrap the key from the PGP key. - """ - function = 'pgp_pub_decrypt' - template = "%(function)s(%(field)s, dearmor('{}'))".format( - settings.PRIVATE_PGP_KEY, - ) - - -class PGPSymmetricKeySQL: - """Custom SQL aggregate to decrypt a field with public key. - - `PGPSymmetricKeySQL` provides a SQL template using pgcrypto to decrypt - data from a field in the database. - - `function` defines `pgp_sym_decrypt` which is a pgcrypto SQL function. - This function takes two arguments: - - a encrypted message (bytea); - - a key (bytea). - - `%(function)s` in `template` is populated by `sql_function`. - - `%(field)s` is replaced with the field's name. - """ - function = 'pgp_sym_decrypt' - template = "%(function)s(%(field)s, '{}')".format( - settings.PGCRYPTO_KEY, - ) - - -class PGPPublicKeyAggregate(PGPPublicKeySQL, Aggregate): - """PGP public key based aggregation. - - `pgp_pub_encrypt` and `dearmor` are pgcrypto functions which encrypt - the field's value with the PGP key unwrapped by `dearmor`. - """ - name = 'decrypted' - - -class PGPSymmetricKeyAggregate(PGPSymmetricKeySQL, Aggregate): - """PGP symmetric key based aggregation. - - `pgp_sym_encrypt` is a pgcrypto functions, encrypts the field's value - with a key. - """ - name = 'decrypted' - - -class DatePGPPublicKeyAggregate(Aggregate): - """PGP public key based aggregation. - - `pgp_sym_encrypt` is a pgcrypto functions, encrypts the field's value - with a key. - """ - name = 'decrypted' - sql = PGPPublicKeySQL - function = 'pgp_pub_decrypt' - template = "cast(%(function)s(%(field)s, dearmor('{}')) AS DATE)".format( - settings.PRIVATE_PGP_KEY, - ) - - -class DatePGPSymmetricKeyAggregate(Aggregate): - """PGP symmetric key based aggregation. - - `pgp_sym_encrypt` is a pgcrypto functions, encrypts the field's value - with a key. - """ - name = 'decrypted' - sql = PGPSymmetricKeySQL - function = 'pgp_sym_decrypt' - template = "cast(%(function)s(%(field)s, '{}') AS DATE)".format( - settings.PGCRYPTO_KEY, - ) - - -class DateTimePGPPublicKeyAggregate(Aggregate): - """PGP public key based aggregation. - - `pgp_sym_encrypt` is a pgcrypto functions, encrypts the field's value - with a key. - """ - name = 'decrypted' - sql = PGPPublicKeySQL - function = 'pgp_pub_decrypt' - template = "cast(%(function)s(%(field)s, dearmor('{}')) AS TIMESTAMP)".format( - settings.PRIVATE_PGP_KEY, - ) - - -class DateTimePGPSymmetricKeyAggregate(Aggregate): - """PGP symmetric key based aggregation. - - `pgp_sym_encrypt` is a pgcrypto functions, encrypts the field's value - with a key. - """ - name = 'decrypted' - sql = PGPSymmetricKeySQL - function = 'pgp_sym_decrypt' - template = "cast(%(function)s(%(field)s, '{}') AS TIMESTAMP)".format( - settings.PGCRYPTO_KEY, - ) diff --git a/pgcrypto/fields.py b/pgcrypto/fields.py index 77e5e20..657ea34 100644 --- a/pgcrypto/fields.py +++ b/pgcrypto/fields.py @@ -51,6 +51,7 @@ class EmailPGPPublicKeyField(EmailPGPPublicKeyFieldMixin, models.EmailField): class IntegerPGPPublicKeyField(PGPPublicKeyFieldMixin, models.IntegerField): """Integer PGP public key encrypted field.""" encrypt_sql = INTEGER_PGP_PUB_ENCRYPT_SQL + cast_type = 'INT4' class TextPGPPublicKeyField(PGPPublicKeyFieldMixin, models.TextField): @@ -88,6 +89,7 @@ class EmailPGPSymmetricKeyField(EmailPGPSymmetricKeyFieldMixin, models.EmailFiel class IntegerPGPSymmetricKeyField(PGPSymmetricKeyFieldMixin, models.IntegerField): """Integer PGP symmetric key encrypted field.""" encrypt_sql = INTEGER_PGP_SYM_ENCRYPT_SQL + cast_type = 'INT4' class TextPGPSymmetricKeyField(PGPSymmetricKeyFieldMixin, models.TextField): diff --git a/pgcrypto/lookups.py b/pgcrypto/lookups.py index c9c2488..bcf9bf3 100644 --- a/pgcrypto/lookups.py +++ b/pgcrypto/lookups.py @@ -33,14 +33,12 @@ def as_sql(self, qn, connection): lhs, params = self.process_lhs(qn, connection) rhs, rhs_params = self.process_rhs(qn, connection) params.extend(rhs_params) - lhs = self.lhs.field.decrypt_sql % lhs - lhs = self.lhs.field.cast_sql % lhs rhs = self.get_rhs_op(connection, rhs) return "%s %s" % (lhs, rhs), params def get_rhs_op(self, connection, rhs): """Build right hand SQL with operator.""" - return '%s %s' % (self.operator, rhs) + return "%s %s" % (self.operator, rhs) class DateTimeGtLookup(DateTimeLookupBase): diff --git a/pgcrypto/managers.py b/pgcrypto/managers.py deleted file mode 100644 index 2476735..0000000 --- a/pgcrypto/managers.py +++ /dev/null @@ -1,43 +0,0 @@ -from django.db import models - -from pgcrypto.mixins import PGPMixin - - -class PGPManager(models.Manager): - use_for_related_fields = True - use_in_migrations = True - - @staticmethod - def _get_pgp_decrypt_sql(field): - """Decrypt sql for symmetric fields using the cast sql if required.""" - name = '"{0}"."{1}"'.format(field.model._meta.db_table, field.name) - sql = field.decrypt_sql % name - if hasattr(field, 'cast_sql') and field.cast_sql: - sql = field.cast_sql % sql - - return sql - - def get_queryset(self, *args, **kwargs): - """Decryption in queryset through meta programming.""" - skip_decrypt = kwargs.pop('skip_decrypt', None) - - qs = super().get_queryset(*args, **kwargs) - - # The Django admin skips this process because it's extremely slow - if not skip_decrypt: - select_sql = {} - encrypted_fields = [] - - for field in self.model._meta.get_fields(): - if isinstance(field, PGPMixin): - select_sql[field.name] = self._get_pgp_decrypt_sql(field) - encrypted_fields.append(field.name) - - # Django queryset.extra() is used here to add decryption sql to query. - qs = qs.defer( - *encrypted_fields - ).extra( - select=select_sql - ) - - return qs diff --git a/pgcrypto/mixins.py b/pgcrypto/mixins.py index 5212702..a32685e 100644 --- a/pgcrypto/mixins.py +++ b/pgcrypto/mixins.py @@ -1,4 +1,6 @@ from django.core.validators import MaxLengthValidator +from django.db.models.expressions import Col +from django.utils.functional import cached_property from pgcrypto import ( PGP_PUB_DECRYPT_SQL, @@ -6,16 +8,7 @@ PGP_SYM_DECRYPT_SQL, PGP_SYM_ENCRYPT_SQL, ) -from pgcrypto.aggregates import ( - DatePGPPublicKeyAggregate, - DatePGPSymmetricKeyAggregate, - DateTimePGPPublicKeyAggregate, - DateTimePGPSymmetricKeyAggregate, - PGPPublicKeyAggregate, - PGPSymmetricKeyAggregate, -) from pgcrypto.forms import DateField, DateTimeField -from pgcrypto.proxy import EncryptedProxyField def remove_validators(validators, validator_class): @@ -23,6 +16,24 @@ def remove_validators(validators, validator_class): return [v for v in validators if not isinstance(v, validator_class)] +class DecryptedCol(Col): + """Provide DecryptedCol support without using `extra` sql.""" + + def __init__(self, alias, target, output_field=None): + """Init the decryption.""" + self.decrypt_sql = target.decrypt_sql + self.cast_type = target.cast_type + self.target = target + + super(DecryptedCol, self).__init__(alias, target, output_field) + + def as_sql(self, compiler, connection): + """Build SQL with decryption and casting.""" + sql, params = super(DecryptedCol, self).as_sql(compiler, connection) + sql = self.decrypt_sql % (sql, self.cast_type) + return sql, params + + class HashMixin: """Keyed hash mixin. @@ -63,29 +74,15 @@ class PGPMixin: `PGPMixin` uses 'pgcrypto' to encrypt data in a postgres database. """ - descriptor_class = EncryptedProxyField encrypt_sql = None # Set in implementation class decrypt_sql = None # Set in implementation class - cast_sql = None # Set in implementation class + cast_type = None def __init__(self, *args, **kwargs): """`max_length` should be set to None as encrypted text size is variable.""" kwargs['max_length'] = None super().__init__(*args, **kwargs) - def contribute_to_class(self, cls, name, **kwargs): - """ - Add a decrypted field proxy to the model. - - Add to the field model an `EncryptedProxyField` to get the decrypted - values of the field. - - The decrypted value can be accessed using the field's name attribute on - the model instance. - """ - super().contribute_to_class(cls, name, **kwargs) - setattr(cls, self.name, self.descriptor_class(field=self)) - def db_type(self, connection=None): """Value stored in the database is hexadecimal.""" return 'bytea' @@ -103,19 +100,40 @@ def _check_max_length_attribute(self, **kwargs): """Override `_check_max_length_attribute` to remove check on max_length.""" return [] + def get_col(self, alias, output_field=None): + """Get the decryption for col.""" + if output_field is None: + output_field = self + if alias != self.model._meta.db_table or output_field != self: + return DecryptedCol( + alias, + self, + output_field + ) + else: + return self.cached_col + + @cached_property + def cached_col(self): + """Get cached version of decryption for col.""" + return DecryptedCol( + self.model._meta.db_table, + self + ) + class PGPPublicKeyFieldMixin(PGPMixin): """PGP public key encrypted field mixin for postgres.""" - aggregate = PGPPublicKeyAggregate encrypt_sql = PGP_PUB_ENCRYPT_SQL decrypt_sql = PGP_PUB_DECRYPT_SQL + cast_type = 'TEXT' class PGPSymmetricKeyFieldMixin(PGPMixin): """PGP symmetric key encrypted field mixin for postgres.""" - aggregate = PGPSymmetricKeyAggregate encrypt_sql = PGP_SYM_ENCRYPT_SQL decrypt_sql = PGP_SYM_DECRYPT_SQL + cast_type = 'TEXT' class RemoveMaxLengthValidatorMixin: @@ -131,14 +149,15 @@ class EmailPGPPublicKeyFieldMixin(PGPPublicKeyFieldMixin, RemoveMaxLengthValidat class EmailPGPSymmetricKeyFieldMixin( - PGPSymmetricKeyFieldMixin, RemoveMaxLengthValidatorMixin): + PGPSymmetricKeyFieldMixin, + RemoveMaxLengthValidatorMixin +): """Email mixin for PGP symmetric key fields.""" class DatePGPPublicKeyFieldMixin(PGPPublicKeyFieldMixin): """Date mixin for PGP public key fields.""" - aggregate = DatePGPPublicKeyAggregate - cast_sql = 'cast(%s as DATE)' + cast_type = 'DATE' def formfield(self, **kwargs): """Override the form field with custom PGP DateField.""" @@ -149,8 +168,7 @@ def formfield(self, **kwargs): class DatePGPSymmetricKeyFieldMixin(PGPSymmetricKeyFieldMixin): """Date mixin for PGP symmetric key fields.""" - aggregate = DatePGPSymmetricKeyAggregate - cast_sql = 'cast(%s as DATE)' + cast_type = 'DATE' def formfield(self, **kwargs): """Override the form field with custom PGP DateField.""" @@ -161,8 +179,7 @@ def formfield(self, **kwargs): class DateTimePGPPublicKeyFieldMixin(PGPPublicKeyFieldMixin): """DateTime mixin for PGP public key fields.""" - aggregate = DateTimePGPPublicKeyAggregate - cast_sql = 'cast(%s as TIMESTAMP)' + cast_type = 'TIMESTAMP' def formfield(self, **kwargs): """Override the form field with custom PGP DateTimeField.""" @@ -173,8 +190,7 @@ def formfield(self, **kwargs): class DateTimePGPSymmetricKeyFieldMixin(PGPSymmetricKeyFieldMixin): """DateTime mixin for PGP symmetric key fields.""" - aggregate = DateTimePGPSymmetricKeyAggregate - cast_sql = 'cast(%s as TIMESTAMP)' + cast_type = 'TIMESTAMP' def formfield(self, **kwargs): """Override the form field with custom PGP DateTimeField.""" diff --git a/pgcrypto/proxy.py b/pgcrypto/proxy.py deleted file mode 100644 index c2d59c0..0000000 --- a/pgcrypto/proxy.py +++ /dev/null @@ -1,50 +0,0 @@ -class EncryptedProxyField: - """Descriptor for encrypted values. - - Decrypted values will query the database through the field's model. - - When accessing the field name attribute on a model instance we are - generating N+1 queries. - """ - def __init__(self, field): - """ - Create a proxy for a django field. - - `field` is a django field. - """ - self.field = field - self.model = field.model - self.aggregate = field.aggregate - - def __get__(self, instance, owner=None): - """ - Retrieve the value of the field from the instance. - - If the value has been saved to the database, decrypt it using an aggregate query. - """ - if not instance: - return self - - if not instance.pk: - return instance.__dict__[self.field.name] - - # Value assigned from `__set__` - value = instance.__dict__[self.field.name] - - if isinstance(value, str): - return value - - if isinstance(value, memoryview): - kwargs = {self.field.name: self.aggregate(self.field.name)} - kw_value = self.model.objects.filter(pk=instance.pk).aggregate(**kwargs) - instance.__dict__[self.field.name] = kw_value[self.field.name] - - return instance.__dict__[self.field.name] - - def __set__(self, instance, value): - """ - Store a value in the model instance's __dict__. - - The value will be keyed by the field's name. - """ - instance.__dict__[self.field.name] = value diff --git a/requirements.txt b/requirements.txt index 8042a01..aef03ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,2 @@ -e . -colour-runner==0.1.1 -coverage==4.5.1 -dj-database-url==0.5.0 -django>=1.10,<2.2 -factory-boy==2.11.1 -flake8-docstrings==1.3.0 -flake8-import-order==0.18 -flake8==3.5.0 -incuna-test-utils==6.6.0 -psycopg2-binary==2.7.5 -pyflakes==1.6.0 -pycodestyle==2.3.1 \ No newline at end of file +django>=1.10,<2.2 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..b2f3254 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,15 @@ +colour-runner==0.1.1 +coverage==4.5.1 +dj-database-url==0.5.0 +factory-boy==2.11.1 +flake8-docstrings==1.3.0 +flake8-import-order==0.18 +flake8==3.5.0 +incuna-test-utils==6.6.0 +pip==18.1 +psycopg2-binary==2.7.5 +pyflakes==1.6.0 +pycodestyle==2.3.1 +setuptools==39.1.0 +twine==1.12.1 +wheel==0.32.1 diff --git a/setup.py b/setup.py index 7592a85..621d5bd 100644 --- a/setup.py +++ b/setup.py @@ -18,13 +18,18 @@ have pip >= 9.0 and setuptools >= 24.2, then try again: $ python -m pip install --upgrade pip setuptools $ python -m pip install django-pgcrypto-fields -This will install the latest version of Django which works on your -version of Python. """.format(*(REQUIRED_PYTHON + CURRENT_PYTHON))) sys.exit(1) -version = '2.3.0' +with open('README.md') as readme_file: + readme = readme_file.read() + + +with open('CHANGELOG.md') as changelog_file: + changelog = changelog_file.read() + +version = '2.4.0' setup( name='django-pgcrypto-fields', @@ -33,7 +38,9 @@ version=version, python_requires='>={}.{}'.format(*REQUIRED_PYTHON), license='BSD', - description='Encrypted fields dealing with pgcrypto postgres extension.', + description='Encrypted fields for Django dealing with pgcrypto postgres extension.', + long_description=readme + '\n\n' + changelog, + long_description_content_type='text/markdown', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Framework :: Django', @@ -52,4 +59,5 @@ author='Incuna Ltd', author_email='admin@incuna.com', url='https://github.com/incuna/django-pgcrypto-fields', + test_suite='tests', ) diff --git a/tests/factories.py b/tests/factories.py index 2d46499..2422cc7 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -2,7 +2,16 @@ import factory -from .models import EncryptedModel +from .models import EncryptedFKModel, EncryptedModel + + +class EncryptedFKModelFactory(factory.DjangoModelFactory): + """Factory to generate foreign key data.""" + fk_pgp_sym_field = factory.Sequence('Text with symmetric key {}'.format) + + class Meta: + """Sets up meta for test factory.""" + model = EncryptedFKModel class EncryptedModelFactory(factory.DjangoModelFactory): @@ -25,6 +34,8 @@ class EncryptedModelFactory(factory.DjangoModelFactory): date_pgp_sym_field = date.today() datetime_pgp_sym_field = datetime.now() + fk_model = factory.SubFactory(EncryptedFKModelFactory) + class Meta: """Sets up meta for test factory.""" model = EncryptedModel diff --git a/tests/models.py b/tests/models.py index b6b1d65..7571bf7 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,10 +1,22 @@ from django.db import models -from pgcrypto import fields, managers +from pgcrypto import fields -class EncryptedModelManager(managers.PGPManager): - pass +class EncryptedFKModel(models.Model): + """Dummy model used to test FK decryption.""" + fk_pgp_sym_field = fields.TextPGPSymmetricKeyField(blank=True, null=True) + + class Meta: + """Sets up the meta for the test model.""" + app_label = 'tests' + + +class EncryptedModelManager(models.Manager): + + def get_by_natural_key(self, email_pgp_pub_field): + """Get by natual key of email pub field.""" + return self.get(email_pgp_pub_field=email_pgp_pub_field) class EncryptedModel(models.Model): @@ -16,7 +28,8 @@ class EncryptedModel(models.Model): hmac_with_original_field = fields.TextHMACField(blank=True, null=True, original='pgp_sym_field') - email_pgp_pub_field = fields.EmailPGPPublicKeyField(blank=True, null=True) + email_pgp_pub_field = fields.EmailPGPPublicKeyField(blank=True, null=True, + unique=True) integer_pgp_pub_field = fields.IntegerPGPPublicKeyField(blank=True, null=True) pgp_pub_field = fields.TextPGPPublicKeyField(blank=True, null=True) date_pgp_pub_field = fields.DatePGPPublicKeyField(blank=True, null=True) @@ -27,16 +40,27 @@ class EncryptedModel(models.Model): pgp_sym_field = fields.TextPGPSymmetricKeyField(blank=True, null=True) date_pgp_sym_field = fields.DatePGPSymmetricKeyField(blank=True, null=True) datetime_pgp_sym_field = fields.DateTimePGPSymmetricKeyField(blank=True, null=True) + fk_model = models.ForeignKey( + EncryptedFKModel, blank=True, null=True, on_delete=models.CASCADE + ) + + objects = EncryptedModelManager() class Meta: """Sets up the meta for the test model.""" app_label = 'tests' -class EncryptedModelWithManager(EncryptedModel): +class EncryptedDateTime(models.Model): + value = fields.DateTimePGPSymmetricKeyField() - objects = EncryptedModelManager() - class Meta: - """Sets up the meta for the test manager.""" - proxy = True +class RelatedDateTime(models.Model): + related = models.ForeignKey( + EncryptedDateTime, + on_delete=models.CASCADE, + related_name='related') + related_again = models.ForeignKey( + EncryptedDateTime, null=True, + on_delete=models.CASCADE, related_name='related_again' + ) diff --git a/tests/run.py b/tests/run.py index 88b4ea4..31f7e15 100755 --- a/tests/run.py +++ b/tests/run.py @@ -29,10 +29,7 @@ PRIVATE_PGP_KEY=open(PRIVATE_PGP_KEY_PATH, 'r').read(), PGCRYPTO_KEY='ultrasecret', ) - - -if django.VERSION >= (1, 7): - django.setup() +django.setup() class TestRunner(ColourRunnerMixin, DiscoverRunner): diff --git a/tests/test_fields.py b/tests/test_fields.py index ee8dd6a..636e051 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,13 +1,15 @@ from datetime import date, datetime from unittest.mock import MagicMock +from django import VERSION as DJANGO_VERSION +from django.db import models from django.test import TestCase from incuna_test_utils.utils import field_names -from pgcrypto import aggregates, fields, proxy -from .factories import EncryptedModelFactory +from pgcrypto import fields +from .factories import EncryptedFKModelFactory, EncryptedModelFactory from .forms import EncryptedForm -from .models import EncryptedModel, EncryptedModelWithManager +from .models import EncryptedDateTime, EncryptedFKModel, EncryptedModel, RelatedDateTime KEYED_FIELDS = (fields.TextDigestField, fields.TextHMACField) EMAIL_PGP_FIELDS = (fields.EmailPGPPublicKeyField, fields.EmailPGPSymmetricKeyField) @@ -86,6 +88,7 @@ def test_fields(self): 'datetime_pgp_sym_field', 'date_pgp_pub_field', 'datetime_pgp_pub_field', + 'fk_model', ) self.assertCountEqual(fields, expected) @@ -109,26 +112,18 @@ def test_value_returned_is_not_bytea(self): self.assertIsInstance(instance.date_pgp_sym_field, date) self.assertIsInstance(instance.datetime_pgp_sym_field, datetime) - def test_fields_descriptor_is_not_instance(self): - """`EncryptedProxyField` instance returns itself when accessed from the model.""" - self.assertIsInstance( - self.model.pgp_pub_field, - proxy.EncryptedProxyField, - ) - self.assertIsInstance( - self.model.pgp_sym_field, - proxy.EncryptedProxyField, - ) - def test_value_query(self): - """Assert querying the field's value is making one query.""" + """Assert querying the field's value is making zero queries.""" expected = 'bonjour' + temp = None EncryptedModelFactory.create(pgp_pub_field=expected) instance = self.model.objects.get() - with self.assertNumQueries(1): - instance.pgp_pub_field + with self.assertNumQueries(0): + temp = instance.pgp_pub_field + + self.assertEqual(expected, temp) def test_value_pgp_pub(self): """Assert we can get back the decrypted value.""" @@ -168,34 +163,33 @@ def test_instance_not_saved(self): self.assertEqual(instance.pgp_pub_field, expected) self.assertEqual(instance.pgp_pub_field, expected) - def test_decrypt_annotate(self): - """Assert we can get back the decrypted value.""" + def test_decrypt_filter(self): + """Assert we can get filter the decrypted value.""" expected = 'bonjour' EncryptedModelFactory.create( pgp_pub_field=expected, - pgp_sym_field=expected, ) - queryset = self.model.objects.annotate( - aggregates.PGPPublicKeyAggregate('pgp_pub_field'), - aggregates.PGPSymmetricKeyAggregate('pgp_sym_field'), + queryset = self.model.objects.filter( + pgp_pub_field=expected ) - instance = queryset.get() - self.assertEqual(instance.pgp_pub_field__decrypted, expected) - self.assertEqual(instance.pgp_sym_field__decrypted, expected) - def test_decrypt_filter(self): - """Assert we can get filter the decrypted value.""" - expected = 'bonjour' - EncryptedModelFactory.create( - pgp_pub_field=expected, + instance = queryset.first() + self.assertEqual(instance.pgp_pub_field, expected) + + queryset = self.model.objects.filter( + pgp_pub_field__contains='jour' ) - queryset = self.model.objects.annotate( - aggregates.PGPPublicKeyAggregate('pgp_pub_field'), + instance = queryset.first() + self.assertEqual(instance.pgp_pub_field, expected) + + queryset = self.model.objects.filter( + pgp_pub_field__startswith='bon' ) - instance = queryset.filter(pgp_pub_field__decrypted=expected).first() - self.assertEqual(instance.pgp_pub_field__decrypted, expected) + + instance = queryset.first() + self.assertEqual(instance.pgp_pub_field, expected) def test_digest_lookup(self): """Assert we can filter a digest value.""" @@ -796,47 +790,245 @@ def test_null(self): instance = EncryptedModel.objects.create() fields = field_names(self.model) fields.remove('id') + for field in fields: - with self.subTest(field=field): - self.assertEqual(getattr(instance, field), None) + with self.subTest(instance=instance, field=field): + value = getattr(instance, field) + self.assertEqual( + value, + None, + msg='Field {}, Value: {}'.format(field, value) + ) + + def test_defer(self): + """Test defer() functionality.""" + expected = 'bonjour' + EncryptedModelFactory.create(pgp_sym_field=expected) + instance = self.model.objects.defer('pgp_sym_field').get() + # Assert that accessing a field that is in defer() causes a query + with self.assertNumQueries(1): + temp = instance.pgp_sym_field -class TestPGPManager(TestCase): - """Test `PGPManager` can be integrated in a `Django` model.""" - model = EncryptedModelWithManager + self.assertEqual(temp, expected) - def test_auto_decryption(self): - """Assert auto decryption via manager.""" - expected_string = 'bonjour' - expected_date = date(2016, 9, 1) - expected_datetime = datetime(2016, 9, 1, 0, 0, 0) + def test_only(self): + """Test only() functionality.""" + expected = 'bonjour' + EncryptedModelFactory.create(pgp_sym_field=expected, pgp_pub_field=expected) + instance = self.model.objects.only('pgp_sym_field').get() - EncryptedModelFactory.create( - digest_field=expected_string, - hmac_field=expected_string, - pgp_pub_field=expected_string, - pgp_sym_field=expected_string, - date_pgp_sym_field=expected_date, # Tests cast sql - datetime_pgp_sym_field=expected_datetime, # Tests cast sql + # Assert that accessing a field in only() does not cause a query + with self.assertNumQueries(0): + temp = instance.pgp_sym_field + + self.assertEqual(temp, expected) + + # Assert that accessing a field not in only() causes a query + with self.assertNumQueries(1): + temp = instance.pgp_pub_field + + self.assertEqual(temp, expected) + + def test_fk_auto_decryption(self): + """Test auto decryption of FK when select related is defined.""" + expected = 'bonjour' + EncryptedModelFactory.create(fk_model__fk_pgp_sym_field=expected) + instance = self.model.objects.select_related('fk_model').get() + + # Assert no additional queries are made to decrypt + with self.assertNumQueries(0): + temp = instance.fk_model.fk_pgp_sym_field + + self.assertEqual(temp, expected) + + def test_get_by_natural_key(self): + """Test get_by_natual_key() support.""" + expected = 'peter@test.com' + EncryptedModelFactory.create(email_pgp_pub_field=expected) + + instance = self.model.objects.get_by_natural_key(expected) + + self.assertEqual(instance.email_pgp_pub_field, expected) + + def test_get_or_create(self): + """Test get_or_create() support.""" + expected = 'peter@test.com' + original = EncryptedModelFactory.create(email_pgp_pub_field=expected) + + instance, created = self.model.objects.get_or_create( + email_pgp_pub_field=expected ) - instance = self.model.objects.get() + self.assertFalse(created) + self.assertEqual(instance.id, original.id) + self.assertEqual(instance.email_pgp_pub_field, original.email_pgp_pub_field) + + instance, created = self.model.objects.get_or_create( + email_pgp_pub_field='jessica@test.com' + ) + + self.assertTrue(created) + self.assertNotEqual(instance.id, original.id) + self.assertEqual(instance.email_pgp_pub_field, 'jessica@test.com') + + def test_update_or_create(self): + """Test update_or_create() support.""" + expected = 'peter@test.com' + original = EncryptedModelFactory.create( + email_pgp_pub_field=expected, + pgp_sym_field='Test' + ) + + instance, created = self.model.objects.update_or_create( + email_pgp_pub_field='jessica@test.com' + ) + + self.assertTrue(created) + self.assertNotEqual(instance.id, original.id) + self.assertEqual(instance.email_pgp_pub_field, 'jessica@test.com') + + instance, created = self.model.objects.update_or_create( + email_pgp_pub_field='jessica@test.com', + defaults={ + 'pgp_sym_field': 'Blue', + } + ) + + self.assertFalse(created) + self.assertNotEqual(instance.id, original.id) + self.assertEqual(instance.pgp_sym_field, 'Blue') + + def test_aggregates(self): + """Test aggregate support.""" + EncryptedModelFactory.create(datetime_pgp_sym_field=datetime(2016, 7, 1, 0, 0, 0)) + EncryptedModelFactory.create(datetime_pgp_sym_field=datetime(2016, 7, 2, 0, 0, 0)) + EncryptedModelFactory.create(datetime_pgp_sym_field=datetime(2016, 8, 1, 0, 0, 0)) + EncryptedModelFactory.create(datetime_pgp_sym_field=datetime(2016, 9, 1, 0, 0, 0)) + EncryptedModelFactory.create(datetime_pgp_sym_field=datetime(2016, 9, 2, 0, 0, 0)) + + total_2016 = self.model.objects.aggregate( + count=models.Count('datetime_pgp_sym_field') + ) + + self.assertEqual(5, total_2016['count']) + + total_july = self.model.objects.filter( + datetime_pgp_sym_field__range=[ + datetime(2016, 7, 1, 0, 0, 0), + datetime(2016, 7, 30, 23, 59, 59) + ] + ).aggregate( + count=models.Count('datetime_pgp_sym_field') + ) + + self.assertEqual(2, total_july['count']) + + total_2016 = self.model.objects.aggregate( + count=models.Count('datetime_pgp_sym_field'), + min=models.Min('datetime_pgp_sym_field'), + max=models.Max('datetime_pgp_sym_field'), + ) + + self.assertEqual(5, total_2016['count']) + self.assertEqual(datetime(2016, 7, 1, 0, 0, 0), total_2016['min']) + self.assertEqual(datetime(2016, 9, 2, 0, 0, 0), total_2016['max']) + + total_july = self.model.objects.filter( + datetime_pgp_sym_field__range=[ + datetime(2016, 7, 1, 0, 0, 0), + datetime(2016, 7, 30, 23, 59, 59) + ] + ).aggregate( + count=models.Count('datetime_pgp_sym_field'), + min=models.Min('datetime_pgp_sym_field'), + max=models.Max('datetime_pgp_sym_field'), + ) + + self.assertEqual(2, total_july['count']) + self.assertEqual(datetime(2016, 7, 1, 0, 0, 0), total_july['min']) + self.assertEqual(datetime(2016, 7, 2, 0, 0, 0), total_july['max']) + + def test_distinct(self): + """Test distinct support.""" + EncryptedModelFactory.create(pgp_sym_field='Paul') + EncryptedModelFactory.create(pgp_sym_field='Paul') + EncryptedModelFactory.create(pgp_sym_field='Peter') + EncryptedModelFactory.create(pgp_sym_field='Peter') + EncryptedModelFactory.create(pgp_sym_field='Jessica') + EncryptedModelFactory.create(pgp_sym_field='Jessica') + + items = self.model.objects.filter( + pgp_sym_field__startswith='P' + ).annotate( + _distinct=models.F('pgp_sym_field') + ).only( + 'id', 'pgp_sym_field', 'fk_model__fk_pgp_sym_field' + ).distinct( + '_distinct' + ) + + self.assertEqual( + 2, + len(items) + ) + + # This only works on Django 2.1+ + if DJANGO_VERSION[0] >= 2 and DJANGO_VERSION[1] >= 1: + items = self.model.objects.filter( + pgp_sym_field__startswith='P' + ).only( + 'id', 'pgp_sym_field', 'fk_model__fk_pgp_sym_field' + ).distinct( + 'pgp_sym_field' + ) + + self.assertEqual( + 2, + len(items) + ) + + def test_annotate(self): + """Test annotate support.""" + efk = EncryptedFKModelFactory.create() + EncryptedModelFactory.create(pgp_sym_field='Paul', fk_model=efk) + EncryptedModelFactory.create(pgp_sym_field='Peter', fk_model=efk) + EncryptedModelFactory.create(pgp_sym_field='Peter', fk_model=efk) + EncryptedModelFactory.create(pgp_sym_field='Jessica', fk_model=efk) + + items = EncryptedFKModel.objects.annotate( + name_count=models.Count('encryptedmodel') + ) + + self.assertEqual( + 4, + items[0].name_count + ) + + items = EncryptedFKModel.objects.filter( + encryptedmodel__pgp_sym_field__startswith='J' + ).annotate( + name_count=models.Count('encryptedmodel') + ) + + self.assertEqual( + 1, + items[0].name_count + ) + + def test_get_col(self): + """Test get_col for related alias.""" + related = EncryptedDateTime.objects.create(value=datetime.now()) + related_again = EncryptedDateTime.objects.create(value=datetime.now()) + + RelatedDateTime.objects.create(related=related, related_again=related_again) + + instance = RelatedDateTime.objects.select_related( + 'related', 'related_again' + ) + + instance = RelatedDateTime.objects.select_related( + 'related', 'related_again' + ).get() - # Using `__dict__` bypasses "on the fly" decryption that normally occurs - # if accessing a field that is not yet decrypted. - # If decryption is not working, we get references to classes - self.assertEqual(instance.__dict__['pgp_pub_field'], expected_string) - self.assertEqual(instance.__dict__['pgp_sym_field'], expected_string) - self.assertEqual(instance.__dict__['date_pgp_sym_field'], expected_date) - self.assertEqual(instance.__dict__['datetime_pgp_sym_field'], expected_datetime) - - # Ensure digest / hmac fields are unaffected - count = self.model.objects.filter( - digest_field__hash_of=expected_string - ).count() - self.assertEqual(count, 1) - - count = self.model.objects.filter( - hmac_field__hash_of=expected_string - ).count() - self.assertEqual(count, 1) + self.assertIsInstance(instance, RelatedDateTime)