From bbd6b627890b61e9e6bcbc1c45586e5f27b6cc7f Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Sun, 1 Nov 2020 01:45:35 -0400 Subject: [PATCH 1/9] Add websocket unittests and packages * Added pytest-timeout in case a websocket is stuck in a while loop * Added unittests for testing websocket connection and pinging * Moved websocket.py to users app since users may want to utilize other Django apps * Added additional "Cython" dependencies to base.txt requirements file to be performant * Added ALLOWED_HOSTS to be several different hosts for websocket testing purposes * Added TODO for developers to add authentication/authorization for websocket handling using the scope. This is currently implemented very nicely with CSRF and Session cache checking at Velnota.com, but the amount of security checking and other files prevents me from adding my knowledge to this PR. It's possible to also use Django Channels, but I still find it infuriating to use sometimes since I support one-socket-per-user connections Signed-off-by: Andrew-Chen-Wang --- hooks/post_gen_project.py | 8 +++- {{cookiecutter.project_slug}}/config/asgi.py | 2 +- .../config/settings/test.py | 5 +++ .../requirements/local.txt | 7 +++- .../users/tests/async_server.py | 40 +++++++++++++++++++ .../users/tests/test_socket.py | 33 +++++++++++++++ .../users}/websocket.py | 15 ++++--- 7 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 {{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py create mode 100644 {{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/test_socket.py rename {{cookiecutter.project_slug}}/{config => {{cookiecutter.project_slug}}/users}/websocket.py (52%) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 50fcbea245..69bf168244 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -125,7 +125,13 @@ def remove_celery_files(): def remove_async_files(): file_names = [ os.path.join("config", "asgi.py"), - os.path.join("config", "websocket.py"), + os.path.join("{{cookiecutter.project_slug}}", "users", "websocket.py"), + os.path.join( + "{{cookiecutter.project_slug}}", "users", "tests", "async_server.py" + ), + os.path.join( + "{{cookiecutter.project_slug}}", "users", "tests", "test_socket.py" + ), ] for file_name in file_names: os.remove(file_name) diff --git a/{{cookiecutter.project_slug}}/config/asgi.py b/{{cookiecutter.project_slug}}/config/asgi.py index 8c99bbf530..8bb2bf9e12 100644 --- a/{{cookiecutter.project_slug}}/config/asgi.py +++ b/{{cookiecutter.project_slug}}/config/asgi.py @@ -28,7 +28,7 @@ # application = HelloWorldApplication(application) # Import websocket application here, so apps from django_application are loaded first -from config.websocket import websocket_application # noqa isort:skip +from {{ cookiecutter.project_slug }}.users.websocket import websocket_application # noqa isort:skip async def application(scope, receive, send): diff --git a/{{cookiecutter.project_slug}}/config/settings/test.py b/{{cookiecutter.project_slug}}/config/settings/test.py index 222597ad9a..87f4d81d21 100644 --- a/{{cookiecutter.project_slug}}/config/settings/test.py +++ b/{{cookiecutter.project_slug}}/config/settings/test.py @@ -15,6 +15,11 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#test-runner TEST_RUNNER = "django.test.runner.DiscoverRunner" +{%- if cookiecutter.use_async == 'y' %} +# Needed for socket testing that also needs HTTP to go along with it +ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"] +{%- endif %} + # PASSWORDS # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#password-hashers diff --git a/{{cookiecutter.project_slug}}/requirements/local.txt b/{{cookiecutter.project_slug}}/requirements/local.txt index 0ddf3496cc..b4f757fcf8 100644 --- a/{{cookiecutter.project_slug}}/requirements/local.txt +++ b/{{cookiecutter.project_slug}}/requirements/local.txt @@ -15,11 +15,14 @@ watchgod==0.7 # https://github.com/samuelcolvin/watchgod # ------------------------------------------------------------------------------ mypy==0.930 # https://github.com/python/mypy django-stubs==1.9.0 # https://github.com/typeddjango/django-stubs -pytest==6.2.5 # https://github.com/pytest-dev/pytest -pytest-sugar==0.9.4 # https://github.com/Frozenball/pytest-sugar {%- if cookiecutter.use_drf == "y" %} djangorestframework-stubs==1.4.0 # https://github.com/typeddjango/djangorestframework-stubs {%- endif %} +pytest==6.2.5 # https://github.com/pytest-dev/pytest +pytest-sugar==0.9.4 # https://github.com/Frozenball/pytest-sugar +{%- if cookiecutter.use_async == 'y' %} +pytest-timeout==1.4.2 # https://github.com/pytest-dev/pytest-timeout/ +{%- endif %} # Documentation # ------------------------------------------------------------------------------ diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py new file mode 100644 index 0000000000..5d4f1248c9 --- /dev/null +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py @@ -0,0 +1,40 @@ +import asyncio +import functools +import threading +import time +from contextlib import contextmanager + +from uvicorn.config import Config +from uvicorn.main import ServerState +from uvicorn.protocols.http.h11_impl import H11Protocol +from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol + + +def run_loop(loop): + loop.run_forever() + loop.close() + + +@contextmanager +def run_server(app, path="/"): + asyncio.set_event_loop(None) + loop = asyncio.new_event_loop() + config = Config(app=app, ws=WebSocketProtocol) + server_state = ServerState() + protocol = functools.partial(H11Protocol, config=config, server_state=server_state) + create_server_task = loop.create_server(protocol, host="127.0.0.1") + server = loop.run_until_complete(create_server_task) + port = server.sockets[0].getsockname()[1] + url = "ws://127.0.0.1:{port}{path}".format(port=port, path=path) + try: + # Run the event loop in a new thread. + thread = threading.Thread(target=run_loop, args=[loop]) + thread.start() + # Return the contextmanager state. + yield url + finally: + # Close the loop from our main thread. + while server_state.tasks: + time.sleep(0.01) + loop.call_soon_threadsafe(loop.stop) + thread.join() diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/test_socket.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/test_socket.py new file mode 100644 index 0000000000..2cb4977397 --- /dev/null +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/test_socket.py @@ -0,0 +1,33 @@ +from asyncio import new_event_loop + +import pytest +from websockets import connect + +from {{cookiecutter.project_slug}}.users.tests.async_server import run_server +from {{cookiecutter.project_slug}}.users.websocket import websocket_application as app + + +def test_accept_connection(): + async def open_connection(url): + async with connect(url) as websocket: + return websocket.open + + with run_server(app) as _url: + loop = new_event_loop() + is_open = loop.run_until_complete(open_connection(_url)) + assert is_open + loop.close() + + +@pytest.mark.timeout(10) +def test_ping(): + async def open_connection(url): + async with connect(url) as websocket: + await websocket.send("ping") + return await websocket.recv() + + with run_server(app) as _url: + loop = new_event_loop() + received_message = loop.run_until_complete(open_connection(_url)) + assert received_message == "pong" + loop.close() diff --git a/{{cookiecutter.project_slug}}/config/websocket.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/websocket.py similarity index 52% rename from {{cookiecutter.project_slug}}/config/websocket.py rename to {{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/websocket.py index 81adfbc664..213fb04188 100644 --- a/{{cookiecutter.project_slug}}/config/websocket.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/websocket.py @@ -1,13 +1,18 @@ async def websocket_application(scope, receive, send): + event = await receive() + if event["type"] == "websocket.connect": + # TODO Add authentication by reading scope + # and getting sessionid from cookie + await send({"type": "websocket.accept"}) + else: + await send({"type": "websocket.close"}) + return + while True: event = await receive() - - if event["type"] == "websocket.connect": - await send({"type": "websocket.accept"}) - if event["type"] == "websocket.disconnect": break if event["type"] == "websocket.receive": if event["text"] == "ping": - await send({"type": "websocket.send", "text": "pong!"}) + await send({"type": "websocket.send", "text": "pong"}) From 7a9bbd6f04c448b29298d91d286c0eafa5b61e55 Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Sun, 1 Nov 2020 01:05:39 -0500 Subject: [PATCH 2/9] Test async in cookiecutter's Travis Signed-off-by: Andrew-Chen-Wang --- tests/test_bare.sh | 2 +- tests/test_docker.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bare.sh b/tests/test_bare.sh index c3842bd93b..a71625bf2a 100755 --- a/tests/test_bare.sh +++ b/tests/test_bare.sh @@ -11,7 +11,7 @@ mkdir -p .cache/bare cd .cache/bare # create the project using the default settings in cookiecutter.json -cookiecutter ../../ --no-input --overwrite-if-exists use_docker=n $@ +cookiecutter ../../ --no-input --overwrite-if-exists use_docker=n use_async=y $@ cd my_awesome_project # Install OS deps diff --git a/tests/test_docker.sh b/tests/test_docker.sh index f554e866bc..6b357b57da 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -11,7 +11,7 @@ mkdir -p .cache/docker cd .cache/docker # create the project using the default settings in cookiecutter.json -cookiecutter ../../ --no-input --overwrite-if-exists use_docker=y $@ +cookiecutter ../../ --no-input --overwrite-if-exists use_docker=y use_async=y $@ cd my_awesome_project # Lint by running pre-commit on all files From 1fecdd1d3cb18176150b5b46a939bd14d56d93c3 Mon Sep 17 00:00:00 2001 From: Andrew-Chen-Wang Date: Sun, 1 Nov 2020 01:17:28 -0500 Subject: [PATCH 3/9] Use websockets instead of WSProto * The problem I found (that's why the unittest failed) was that WSProto was not denying an incoming connection if my session authentication failed. Instead, WSProto didn't understand Uvicorn which led to an exception raised. Once https://github.com/encode/uvicorn/issues/811 is fixed, we can switch back to WSProto since websockets seems unmaintained Signed-off-by: Andrew-Chen-Wang --- {{cookiecutter.project_slug}}/requirements/base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/{{cookiecutter.project_slug}}/requirements/base.txt b/{{cookiecutter.project_slug}}/requirements/base.txt index 083e88d45e..ed47e7aba7 100644 --- a/{{cookiecutter.project_slug}}/requirements/base.txt +++ b/{{cookiecutter.project_slug}}/requirements/base.txt @@ -25,6 +25,7 @@ flower==1.0.0 # https://github.com/mher/flower {%- endif %} {%- if cookiecutter.use_async == 'y' %} uvicorn[standard]==0.16.0 # https://github.com/encode/uvicorn +websockets==10.1 # https://github.com/aaugustin/websockets {%- endif %} # Django From 8c2ba0e58ea2cd96444729594cc2ae6c26c2c675 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Sun, 1 Nov 2020 09:31:22 -0500 Subject: [PATCH 4/9] Ignore mypy for async server fixture --- .../{{cookiecutter.project_slug}}/users/tests/async_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py index 5d4f1248c9..6e449d5c6e 100644 --- a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py @@ -23,7 +23,7 @@ def run_server(app, path="/"): server_state = ServerState() protocol = functools.partial(H11Protocol, config=config, server_state=server_state) create_server_task = loop.create_server(protocol, host="127.0.0.1") - server = loop.run_until_complete(create_server_task) + server = loop.run_until_complete(create_server_task) # type: ignore port = server.sockets[0].getsockname()[1] url = "ws://127.0.0.1:{port}{path}".format(port=port, path=path) try: From ad80fc2fdad9a70c2e76cd5ddde5959e423874eb Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Sun, 1 Nov 2020 09:36:06 -0500 Subject: [PATCH 5/9] Ignore type in port --- .../{{cookiecutter.project_slug}}/users/tests/async_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py index 6e449d5c6e..0d8240ee4a 100644 --- a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py @@ -23,8 +23,8 @@ def run_server(app, path="/"): server_state = ServerState() protocol = functools.partial(H11Protocol, config=config, server_state=server_state) create_server_task = loop.create_server(protocol, host="127.0.0.1") - server = loop.run_until_complete(create_server_task) # type: ignore - port = server.sockets[0].getsockname()[1] + server = loop.run_until_complete(create_server_task) + port = server.sockets[0].getsockname()[1] # type: ignore url = "ws://127.0.0.1:{port}{path}".format(port=port, path=path) try: # Run the event loop in a new thread. From 0f86c6424e11ee9995643b19c16057f7a1d400a6 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Wed, 4 Nov 2020 11:31:28 -0500 Subject: [PATCH 6/9] Remove trailing whitespace in async_server.py --- .../{{cookiecutter.project_slug}}/users/tests/async_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py index 0d8240ee4a..07786293a6 100644 --- a/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py +++ b/{{cookiecutter.project_slug}}/{{cookiecutter.project_slug}}/users/tests/async_server.py @@ -23,7 +23,7 @@ def run_server(app, path="/"): server_state = ServerState() protocol = functools.partial(H11Protocol, config=config, server_state=server_state) create_server_task = loop.create_server(protocol, host="127.0.0.1") - server = loop.run_until_complete(create_server_task) + server = loop.run_until_complete(create_server_task) port = server.sockets[0].getsockname()[1] # type: ignore url = "ws://127.0.0.1:{port}{path}".format(port=port, path=path) try: From 3b01a65c1a282b716b4190c5753425482c99cf29 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Fri, 6 Nov 2020 15:39:03 -0500 Subject: [PATCH 7/9] Remove use_async in script --- tests/test_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docker.sh b/tests/test_docker.sh index 6b357b57da..f554e866bc 100755 --- a/tests/test_docker.sh +++ b/tests/test_docker.sh @@ -11,7 +11,7 @@ mkdir -p .cache/docker cd .cache/docker # create the project using the default settings in cookiecutter.json -cookiecutter ../../ --no-input --overwrite-if-exists use_docker=y use_async=y $@ +cookiecutter ../../ --no-input --overwrite-if-exists use_docker=y $@ cd my_awesome_project # Lint by running pre-commit on all files From 917aae328e086570bbd84a71b73020b55774e485 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Fri, 6 Nov 2020 15:39:18 -0500 Subject: [PATCH 8/9] Remove use_async in script --- tests/test_bare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bare.sh b/tests/test_bare.sh index a71625bf2a..c3842bd93b 100755 --- a/tests/test_bare.sh +++ b/tests/test_bare.sh @@ -11,7 +11,7 @@ mkdir -p .cache/bare cd .cache/bare # create the project using the default settings in cookiecutter.json -cookiecutter ../../ --no-input --overwrite-if-exists use_docker=n use_async=y $@ +cookiecutter ../../ --no-input --overwrite-if-exists use_docker=n $@ cd my_awesome_project # Install OS deps From 1d08db9ebf37cf2d7682c6771f6e1fce85cf9213 Mon Sep 17 00:00:00 2001 From: Bruno Alla Date: Fri, 24 Dec 2021 10:52:50 +0000 Subject: [PATCH 9/9] Add a new CI job to test the async configuration --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0180184fa7..01ca4eb2e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,8 @@ jobs: fail-fast: false matrix: script: + - name: With Async + args: "use_async=y" - name: With Celery args: "use_celery=y use_compressor=y" - name: With Gulp