Add settings models for services, events, tasks, and the flow run graph #17532

Workflow file for this run

.github/workflows/python-tests.yaml at 6b02e51

	name: Unit tests

	env:
	# enable colored output
	# https://github.com/pytest-dev/pytest/issues/7443
	PY_COLORS: 1

	on:
	pull_request:
	paths:
	- .github/workflows/python-tests.yaml
	- "src/prefect/*/.py"
	- "tests/*/.py"
	- requirements.txt
	- requirements-client.txt
	- requirements-dev.txt
	- setup.cfg
	- Dockerfile
	push:
	branches:
	- main
	paths:
	- .github/workflows/python-tests.yaml
	- "src/prefect/*/.py"
	- "tests/*/.py"
	- requirements.txt
	- requirements-client.txt
	- requirements-dev.txt
	- setup.cfg
	- Dockerfile

	permissions:
	contents: read
	actions: write

	# Limit concurrency by workflow/branch combination.
	#
	# For pull request builds, pushing additional changes to the
	# branch will cancel prior in-progress and pending builds.
	#
	# For builds triggered on a branch push, additional changes
	# will wait for prior builds to complete before starting.
	#
	# https://docs.github.com/en/actions/using-jobs/using-concurrency
	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	run-tests:
	runs-on:
	group: oss-larger-runners
	name: ${{ matrix.test-type.name }} - python:${{ matrix.python-version }}, ${{ matrix.database }}
	strategy:
	matrix:
	test-type:
	- name: Server Tests
	modules: tests/server/ tests/events/server
	- name: Client Tests
	modules: tests/ --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers
	- name: Runner and Worker Tests
	modules: tests/test_task_runners.py tests/runner tests/workers
	database:
	- "postgres:14"
	- "sqlite"
	python-version:
	- "3.9"
	- "3.10"
	- "3.11"
	- "3.12"
	exclude:
	- database: "sqlite"
	test-type:
	name: Client Tests
	modules: tests/ --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers
	- database: "sqlite"
	test-type:
	name: Runner and Worker Tests
	modules: tests/test_task_runners.py tests/runner tests/workers

	fail-fast: true

	timeout-minutes: 10

	steps:
	- name: Display current test matrix
	run: echo '${{ toJSON(matrix) }}'

	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	fetch-depth: 0

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	id: setup_python
	with:
	python-version: ${{ matrix.python-version }}

	- name: UV Cache
	# Manually cache the uv cache directory
	# until setup-python supports it:
	# https://github.com/actions/setup-python/issues/822
	uses: actions/cache@v4
	id: cache-uv
	with:
	path: ~/.cache/uv
	key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }}

	- name: Install packages
	run: \|
	python -m pip install -U uv
	uv pip install --upgrade --system -e .[dev]

	- name: Start database container
	if: ${{ startsWith(matrix.database, 'postgres') }}
	run: >
	docker run
	--name "postgres"
	--detach
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--publish 5432:5432
	--tmpfs /var/lib/postgresql/data
	--env POSTGRES_USER="prefect"
	--env POSTGRES_PASSWORD="prefect"
	--env POSTGRES_DB="prefect"
	--env LANG="C.UTF-8"
	--env LANGUAGE="C.UTF-8"
	--env LC_ALL="C.UTF-8"
	--env LC_COLLATE="C.UTF-8"
	--env LC_CTYPE="C.UTF-8"
	${{ matrix.database }}
	-c max_connections=250

	./scripts/wait-for-healthy-container.sh postgres 30

	echo "PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

	- name: Start redis
	run: >
	docker run
	--name "redis"
	--detach
	--publish 6379:6379
	redis:latest

	- name: Set coverage file and artifact name
	id: set_coverage_and_artifact_name
	run: \|
	sanitized_test_type="${{ matrix.test-type.name }}"
	sanitized_test_type="${sanitized_test_type// /_}"
	sanitized_database="${{ matrix.database }}"
	sanitized_database="${sanitized_database//:/\-}"
	sanitized_python_version="${{ matrix.python-version }}"
	export COVERAGE_FILE=".coverage.${sanitized_test_type}.${sanitized_python_version}.${sanitized_database}"
	echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV
	echo "artifact_name=coverage-data-${sanitized_test_type}-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT

	- name: Run tests
	run: \|
	echo "Using COVERAGE_FILE=$COVERAGE_FILE"
	pytest ${{ matrix.test-type.modules }} \
	--numprocesses auto \
	--maxprocesses 6 \
	--dist worksteal \
	--disable-docker-image-builds \
	--exclude-service kubernetes \
	--exclude-service docker \
	--durations 26 \
	--cov=prefect \
	--cov-config=setup.cfg \
	--cov-report=''

	- name: Upload coverage data
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }}
	path: ${{ env.COVERAGE_FILE }}
	include-hidden-files: true
	retention-days: 1

	- name: Create and Upload failure flag
	if: ${{ failure() }}
	id: create_failure_flag
	run: \|
	sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}"
	sanitized_name="${sanitized_name//:/-}"
	echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt"
	echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT

	- name: Upload failure flag
	if: ${{ failure() }}
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.create_failure_flag.outputs.artifact_name }}
	path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt"

	- name: Check database container
	# Only applicable for Postgres, but we want this to run even when tests fail
	if: always()
	run: \|
	docker container inspect postgres \
	&& docker container logs postgres \
	\|\| echo "Ignoring bad exit code"

	run-docker-tests:
	runs-on:
	group: oss-larger-runners
	name: docker, python:${{ matrix.python-version }}
	strategy:
	matrix:
	database:
	- "postgres:14"
	python-version:
	- "3.9"
	- "3.10"
	- "3.11"
	- "3.12"

	fail-fast: true

	timeout-minutes: 45

	steps:
	- name: Display current test matrix
	run: echo '${{ toJSON(matrix) }}'

	- uses: actions/checkout@v4
	with:
	persist-credentials: false
	fetch-depth: 0

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	with:
	driver-opts: image=moby/buildkit:v0.12.5

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	id: setup_python
	with:
	python-version: ${{ matrix.python-version }}

	- name: UV Cache
	# Manually cache the uv cache directory
	# until setup-python supports it:
	# https://github.com/actions/setup-python/issues/822
	uses: actions/cache@v4
	id: cache-uv
	with:
	path: ~/.cache/uv
	key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }}

	- name: Get image tag
	id: get_image_tag
	run: \|
	SHORT_SHA=$(git rev-parse --short=7 HEAD)
	tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
	echo "image_tag=${tmp}" >> $GITHUB_OUTPUT

	- name: Login to DockerHub
	uses: docker/login-action@v3
	if: github.event.pull_request.head.repo.full_name == github.repository
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	- name: Build test image
	uses: docker/build-push-action@v6
	env:
	DOCKER_BUILD_SUMMARY: false
	DOCKER_RECORD_UPLOAD: false
	with:
	context: .
	build-args: \|
	PYTHON_VERSION=${{ matrix.python-version }}
	PREFECT_EXTRAS=[dev]
	tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
	outputs: type=docker,dest=/tmp/image.tar

	- name: Test Docker image
	run: \|
	docker load --input /tmp/image.tar
	docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version

	- name: Install packages
	run: \|
	python -m pip install -U uv
	uv pip install --upgrade --system -e .[dev]

	- name: Start database container
	if: ${{ startsWith(matrix.database, 'postgres') }}
	run: >
	docker run
	--name "postgres"
	--detach
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	--publish 5432:5432
	--tmpfs /var/lib/postgresql/data
	--env POSTGRES_USER="prefect"
	--env POSTGRES_PASSWORD="prefect"
	--env POSTGRES_DB="prefect"
	--env LANG="C.UTF-8"
	--env LANGUAGE="C.UTF-8"
	--env LC_ALL="C.UTF-8"
	--env LC_COLLATE="C.UTF-8"
	--env LC_CTYPE="C.UTF-8"
	${{ matrix.database }}
	-c max_connections=250

	./scripts/wait-for-healthy-container.sh postgres 30

	echo "PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

	- name: Start docker registry
	run: >
	docker run
	--name "prefect-test-registry"
	--detach
	--publish 5555:5000
	registry:2

	- name: Start redis
	run: >
	docker run
	--name "redis"
	--detach
	--publish 6379:6379
	redis:latest

	- name: Set coverage file and artifact name
	id: set_coverage_and_artifact_name
	run: \|
	sanitized_database="${{ matrix.database }}"
	sanitized_database="${sanitized_database//:/\-}"
	sanitized_python_version="${{ matrix.python-version }}"
	export COVERAGE_FILE=".coverage.${sanitized_python_version}.${sanitized_database}"
	echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV
	echo "artifact_name=coverage-data-docker-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT

	- name: Run tests
	run: \|
	echo "Using COVERAGE_FILE=$COVERAGE_FILE"
	pytest tests \
	--numprocesses auto \
	--maxprocesses 6 \
	--dist worksteal \
	--disable-docker-image-builds \
	--only-service docker \
	--durations 26 \
	--cov=prefect \
	--cov-config=setup.cfg \
	--cov-report=''

	- name: Upload coverage data
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }}
	path: ${{ env.COVERAGE_FILE }}
	include-hidden-files: true
	retention-days: 1

	- name: Create and Upload failure flag
	if: ${{ failure() }}
	id: create_failure_flag
	run: \|
	sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}"
	sanitized_name="${sanitized_name//:/-}"
	echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt"
	echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT

	- name: Upload failure flag
	if: ${{ failure() }}
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.create_failure_flag.outputs.artifact_name }}
	path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt"

	- name: Check database container
	# Only applicable for Postgres, but we want this to run even when tests fail
	if: always()
	run: \|
	docker container inspect postgres \
	&& docker container logs postgres \
	\|\| echo "Ignoring bad exit code"

	combine-coverage:
	runs-on: ubuntu-latest
	needs:
	- run-tests
	- run-docker-tests
	steps:
	- uses: actions/checkout@v4
	with:
	persist-credentials: false

	- name: Set up Python 3.12
	uses: actions/setup-python@v5
	id: setup_python
	with:
	python-version: "3.12"

	- name: Download coverage data artifacts
	uses: actions/download-artifact@v4
	with:
	pattern: coverage-data-*
	merge-multiple: true

	- name: Install coverage
	run: pip install coverage

	- name: Combine coverage data
	run: coverage combine

	- name: Generate HTML coverage report
	run: coverage html

	- name: Upload combined coverage report
	uses: actions/upload-artifact@v4
	with:
	name: combined-coverage-report
	path: htmlcov/

	- name: Publish coverage markdown report
	run: \|
	echo "## Coverage Report" >> $GITHUB_STEP_SUMMARY
	echo "[Detailed Report](${{ steps.upload_combined_coverage_report.outputs.artifact_url }})" >> $GITHUB_STEP_SUMMARY
	coverage report --format=markdown >> $GITHUB_STEP_SUMMARY

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add settings models for services, events, tasks, and the flow run graph #17532

Workflow file

Add settings models for services, events, tasks, and the flow run graph #17532

Jobs

Run details

Workflow file for this run