From d78a987d25afb621a849e0be42345d089142c521 Mon Sep 17 00:00:00 2001 From: Jeff Hale Date: Mon, 7 Oct 2024 18:30:36 -0400 Subject: [PATCH 1/3] move secret example to store-flow-code and fix GH example --- .../store-flow-code.mdx | 34 +++++++++- .../deploy/run-flows-in-local-processes.mdx | 63 ++++++++----------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx b/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx index 4465ecd62db8..ed64852b03cc 100644 --- a/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx +++ b/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx @@ -63,6 +63,30 @@ If using the Python `deploy` method with a private repository that references a ) ``` + ```python secret_block.py + from prefect import flow + from prefect.runner.storage import GitRepository + from prefect.blocks.system import Secret + + + if __name__ == "__main__": + + github_repo = GitRepository( + url="https://github.com/org/my-private-repo.git", + credentials={ + "access_token": Secret.load("github-access-token") + } + ) + + flow.from_source( + source=github_repo, + entrypoint="gh_credentials_block.py:my_flow", + ).deploy( + name="private-github-deploy", + work_pool_name="my_pool", + ) + ``` + ```python gh_credentials_block.py from prefect import flow from prefect.runner.storage import GitRepository @@ -73,7 +97,7 @@ If using the Python `deploy` method with a private repository that references a github_repo = GitRepository( url="https://github.com/org/my-private-repo.git", - credentials=GitHubCredentials.load("my-github-credentials-block") + credentials=GitHubCredentials.load("my-github-credentials-block"), ) flow.from_source( @@ -95,7 +119,9 @@ If using the Python `deploy` method with a private repository that references a github_repo = GitRepository( url="https://github.com/org/my-private-repo.git", - credentials=Secret.load("my-secret-block-with-my-gh-credentials") + credentials={ + "access_token": Secret.load("my-secret-block-with-my-gh-credentials") + }, ) flow.from_source( @@ -259,6 +285,7 @@ If using the Python `deploy` method with a private repository that references a gitlab_repo = GitRepository( url="https://gitlab.com/org/my-private-repo.git", credentials=GitLabCredentials.load("my-gitlab-credentials-block") + ) flow.from_source( source=gitlab_repo, @@ -319,6 +346,9 @@ If using the Python `deploy` method with a private repository that references a +Note that you can specify a `branch` if creating a `GitRepository` object. +The default is `"main"`. + **Push your code** diff --git a/docs/3.0/deploy/run-flows-in-local-processes.mdx b/docs/3.0/deploy/run-flows-in-local-processes.mdx index 40df8889c625..3f68be8ec887 100644 --- a/docs/3.0/deploy/run-flows-in-local-processes.mdx +++ b/docs/3.0/deploy/run-flows-in-local-processes.mdx @@ -3,8 +3,7 @@ title: Run flows in local processes description: Create a deployment for a flow by calling the `serve` method. --- -The simplest way to create a [deployment](/3.0/deploy/infrastructure-examples/docker/) for your -flow is by calling its `serve` method. +The simplest way to create a [deployment](/3.0/deploy/infrastructure-examples/docker/) for your flow is by calling its `serve` method. ## Serve a flow @@ -29,9 +28,10 @@ if __name__ == "__main__": # generated on the server hello_world.serve(name="my-first-deployment", - tags=["onboarding"], - parameters={"goodbye": True}, - interval=60) + tags=["onboarding"], + parameters={"goodbye": True}, + interval=60 + ) ``` This interface provides the configuration for a deployment (with no @@ -156,8 +156,7 @@ Read more about our [hybrid model](https://www.prefect.io/security/overview/#hyb You can retrieve flows from remote storage with the `flow.from_source` method. -`flow.from_source` accepts a git repository URL and an entrypoint pointing to the -flow to load from the repository: +`flow.from_source` can accept a git repository URL and an entrypoint pointing to the flow to load from the repository: ```python load_from_url.py from prefect import flow @@ -179,38 +178,14 @@ if __name__ == "__main__": 16:40:34.706 | INFO | Flow run 'muscular-perch' - Finished in state Completed() ``` -A flow entrypoint is the path to the file where the flow is located, and the name of the -flow function separated by a colon. - -For additional configuration, such as specifying a private repository, -provide a `GitRepository` object instead of URL: +A flow entrypoint is the path to the file where the flow is located, and the name of the flow function separated by a colon. -```python load_from_storage.py -from prefect import flow -from prefect.runner.storage import GitRepository -from prefect.blocks.system import Secret - - -my_flow = flow.from_source( - source=GitRepository( - url="https://github.com/org/private-repo.git", - branch="dev", - credentials={ - "access_token": Secret.load("github-access-token").get() - } - ), - entrypoint="flows.py:my_flow" -) - - -if __name__ == "__main__": - my_flow() -``` +For more ways to store and access flow code, see the [Retrieve code from storage page](/3.0/deploy/infrastructure-concepts/store-flow-code). **You can serve loaded flows** -You can serve a flow loaded from remote storage with the same [`serve`](#serving-a-flow) method as a local flow: +You can serve a flow loaded from remote storage with the same [`serve`](#serv3-a-flow) method as a local flow: ```python serve_loaded_flow.py from prefect import flow @@ -224,7 +199,19 @@ if __name__ == "__main__": ``` -When you serve a flow loaded from remote storage, the serving process -periodically polls your remote storage for updates to the flow's code. -This pattern allows you to update your flow code without restarting the serving -process. +When you serve a flow loaded from remote storage, the serving process periodically polls your remote storage for updates to the flow's code. +This pattern allows you to update your flow code without restarting the serving process. +Note that if you change the parameters to your flow, you'll need to restart the serving process. + +## If you need dynamic infrastructure + +For more configuration, you can create a deployment that uses a work pool. +Reasons to create a work-pool based deployment include: + +- Wanting to run your flow on dynamically provisioned infrastructure +- Needing more control over the execution environment on a per-flow run basis +- Creating an infrastructure template to use across deployments + +Work pools are popular with data platform teams because they allow you to manage infrastructure configuration across an organization. + +Learn more about work-pool based deployments in the [Configure dynamic infrastructure with work pools page](/3.0/deploy/infrastructure-concepts/work-pools) \ No newline at end of file From dca86e806adca96df2712a4852326d4ca12078eb Mon Sep 17 00:00:00 2001 From: Jeff Hale Date: Mon, 7 Oct 2024 18:31:19 -0400 Subject: [PATCH 2/3] fix typo --- docs/3.0/deploy/run-flows-in-local-processes.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/3.0/deploy/run-flows-in-local-processes.mdx b/docs/3.0/deploy/run-flows-in-local-processes.mdx index 3f68be8ec887..5df76008119d 100644 --- a/docs/3.0/deploy/run-flows-in-local-processes.mdx +++ b/docs/3.0/deploy/run-flows-in-local-processes.mdx @@ -185,7 +185,7 @@ For more ways to store and access flow code, see the [Retrieve code from storage **You can serve loaded flows** -You can serve a flow loaded from remote storage with the same [`serve`](#serv3-a-flow) method as a local flow: +You can serve a flow loaded from remote storage with the same [`serve`](#serve-a-flow) method as a local flow: ```python serve_loaded_flow.py from prefect import flow @@ -214,4 +214,4 @@ Reasons to create a work-pool based deployment include: Work pools are popular with data platform teams because they allow you to manage infrastructure configuration across an organization. -Learn more about work-pool based deployments in the [Configure dynamic infrastructure with work pools page](/3.0/deploy/infrastructure-concepts/work-pools) \ No newline at end of file +Learn more about work-pool based deployments on the [Configure dynamic infrastructure with work pools page](/3.0/deploy/infrastructure-concepts/work-pools) \ No newline at end of file From 366b2b83df920a1740d0d4fa1b736bd225d604b6 Mon Sep 17 00:00:00 2001 From: Jeff Hale Date: Mon, 7 Oct 2024 18:47:23 -0400 Subject: [PATCH 3/3] minor edits --- .../store-flow-code.mdx | 38 +++++-------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx b/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx index ed64852b03cc..43afc6615713 100644 --- a/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx +++ b/docs/3.0/deploy/infrastructure-concepts/store-flow-code.mdx @@ -63,30 +63,6 @@ If using the Python `deploy` method with a private repository that references a ) ``` - ```python secret_block.py - from prefect import flow - from prefect.runner.storage import GitRepository - from prefect.blocks.system import Secret - - - if __name__ == "__main__": - - github_repo = GitRepository( - url="https://github.com/org/my-private-repo.git", - credentials={ - "access_token": Secret.load("github-access-token") - } - ) - - flow.from_source( - source=github_repo, - entrypoint="gh_credentials_block.py:my_flow", - ).deploy( - name="private-github-deploy", - work_pool_name="my_pool", - ) - ``` - ```python gh_credentials_block.py from prefect import flow from prefect.runner.storage import GitRepository @@ -213,7 +189,9 @@ If using the Python `deploy` method with a private repository that references a if __name__ == "__main__": github_repo=GitRepository( url="https://bitbucket.com/org/my-private-repo.git", - credentials=Secret.load("my-secret-block-with-my-bb-credentials") + credentials={ + "access_token": Secret.load("my-secret-block-with-my-bb-credentials") + }, ) flow.from_source( @@ -306,7 +284,9 @@ If using the Python `deploy` method with a private repository that references a if __name__ == "__main__": gitlab_repo = GitRepository( url="https://gitlab.com/org/my-private-repo.git", - credentials=Secret.load("my-secret-block-with-my-gl-credentials") + credentials={ + "access_token": Secret.load("my-secret-block-with-my-gl-credentials") + }, ) flow.from_source( @@ -500,7 +480,7 @@ We also include Python code that shows how to use an existing storage block and ) ``` - ```python s3_blocks.py + ```python s3_block.py from prefect import flow from prefect_aws.s3 import S3Bucket @@ -591,7 +571,7 @@ We also include Python code that shows how to use an existing storage block and ) ``` - ```python azure_blocks.py + ```python azure_block.py from prefect import flow from prefect_azure import AzureBlobCredentials, AzureBlobStorage @@ -678,7 +658,7 @@ We also include Python code that shows how to use an existing storage block and ) ``` - ```python gcs_blocks.py + ```python gcs_block.py from prefect import flow from prefect_gcp import GcpCredentials, GCSBucket