diff --git a/README.md b/README.md index dfed6bd..f7bcc1f 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ Components: * Controller Services: * [OpenTDFControllerService](./nifi-tdf-controller-services-api/src/main/java/io/opentdf/nifi/OpenTDFControllerService.java): A NiFi controller service providing OpenTDF Platform Configuration +## Using a custom TrustStore +Communicating over TLS with self-signed or other untrusted certs can be configured using NiFi's standard [SSL Context Service](https://nifi.apache.org/docs/nifi-docs/components/org.apache.nifi/nifi-ssl-context-service-nar/1.25.0/org.apache.nifi.ssl.StandardSSLContextService/index.html) +and then wired into the processors by setting their respective SSL Context Service properties to use a configured +SSL Context Service. + ## Example See [An Sample NiFi FlowFile Template using ZTDF/NanoTDF Processors](./deploy/Example_ZTDF_NanoTDF.xml) @@ -25,59 +30,21 @@ Upload and use this template in NiFi: #### FlowChart: Generic ZTDF Nifi Flows -```mermaid ---- -title: Generic ZTDF NiFi Flows ---- -flowchart TD - a[Nifi Processor] - b["`**UpdateAttribute**`" Add data policy attributes to FlowFile] - c["`**ConvertToZTDF**`"] - d["Process ZTDF"] - e["Handle Error"] - f[Nifi Processor] - g["`**ConvertFromZTDF**`"] - h[Process Plaintext] - i[Handle Error] - a -- success (content = PlainText) --> b - b -- success (content = PlainText) --> c - c -- success (content = ZTDF) --> d - c -- failure --> e - f -- success (content = ZTDF) --> g - g -- success (content = PlainText) --> h - g -- failure --> i -``` +![diagram](./docs/diagrams/generic_ztdf_nifi_flows.svg) #### FlowChart: Generic NanoTDF NiFi Flows -```mermaid ---- -title: Generic NanoTDF NiFi Flows ---- -flowchart TD - a[Nifi Processor] - b["`**UpdateAttribute**`" Add data policy attributes to FlowFile] - c["`**ConvertToNanoTDF**`"] - d["Process NanoTDF"] - e["Handle Error"] - e2["Handle Max Size Error"] - f[Nifi Processor] - g["`**ConvertFromZTDF**`"] - h[Process Plaintext] - i[Handle Error] - a -- success (content = Plaintext) --> b - b -- success (content = Plaintext)--> c - c -- success (content = NanoTDF) --> d - c -- failure --> e - c -- exceeds_size_limit --> e2 - f -- success (content = NanoTDF) --> g - g -- success (content = Plaintext) --> h - g -- failure --> i -``` + +![diagram](./docs/diagrams/generic_nanotdf_nifi_flows.svg) + # Quick Start - Docker Compose -1. Build the NiFi Archives (NARs) and place in the docker compose mounted volumes +1. Build the NiFi Archives (NARs) and place in the docker compose mounted volumes. The opentd + java-sdk is currently hosted on github's maven package repository, so github credentials are required to perform a maven build. + ```shell + export GITHUB_ACTOR=your gh username + export GITHUB_TOKEN=your gh token make compose-package ``` 1. Start docker compose diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md new file mode 100644 index 0000000..9a337c1 --- /dev/null +++ b/docs/diagrams/README.md @@ -0,0 +1,11 @@ +Mermaid CLI is used to generate images from Mermaid markdown + +[mermaid.cli](https://github.com/mermaid-js/mermaid-cli) +``` +npm install -g @mermaid-js/mermaid-cli +``` + +To generate images from mermaid: +``` +mmdc -i .mmd -o .svg +``` \ No newline at end of file diff --git a/docs/diagrams/generic_nanotdf_nifi_flows.mmd b/docs/diagrams/generic_nanotdf_nifi_flows.mmd new file mode 100644 index 0000000..e14127d --- /dev/null +++ b/docs/diagrams/generic_nanotdf_nifi_flows.mmd @@ -0,0 +1,22 @@ +--- +title: Generic NanoTDF NiFi Flows +--- +flowchart TD + a[Nifi Processor] + b["`**UpdateAttribute**`" Add data policy attributes to FlowFile] + c["`**ConvertToNanoTDF**`"] + d["Process NanoTDF"] + e["Handle Error"] + e2["Handle Max Size Error"] + f[Nifi Processor] + g["`**ConvertFromZTDF**`"] + h[Process Plaintext] + i[Handle Error] + a -- success (content = Plaintext) --> b +b -- success (content = Plaintext)--> c +c -- success (content = NanoTDF) --> d +c -- failure --> e +c -- exceeds_size_limit --> e2 +f -- success (content = NanoTDF) --> g +g -- success (content = Plaintext) --> h +g -- failure --> i \ No newline at end of file diff --git a/docs/diagrams/generic_nanotdf_nifi_flows.svg b/docs/diagrams/generic_nanotdf_nifi_flows.svg new file mode 100644 index 0000000..95d638c --- /dev/null +++ b/docs/diagrams/generic_nanotdf_nifi_flows.svg @@ -0,0 +1,4 @@ +
success (content = Plaintext)
success (content = Plaintext)
success (content = NanoTDF)
failure
exceeds_size_limit
success (content = NanoTDF)
success (content = Plaintext)
failure
Nifi Processor
+

UpdateAttribute Add data policy attributes to FlowFile

+

ConvertToNanoTDF

Process NanoTDF
Handle Error
Handle Max Size Error
Nifi Processor
+

ConvertFromZTDF

Process Plaintext
Handle Error
Generic NanoTDF NiFi Flows \ No newline at end of file diff --git a/docs/diagrams/generic_ztdf_nifi_flows.mmd b/docs/diagrams/generic_ztdf_nifi_flows.mmd new file mode 100644 index 0000000..9822285 --- /dev/null +++ b/docs/diagrams/generic_ztdf_nifi_flows.mmd @@ -0,0 +1,20 @@ +--- +title: Generic ZTDF NiFi Flows +--- +flowchart TD + a[Nifi Processor] + b["`**UpdateAttribute**`" Add data policy attributes to FlowFile] + c["`**ConvertToZTDF**`"] + d["Process ZTDF"] + e["Handle Error"] + f[Nifi Processor] + g["`**ConvertFromZTDF**`"] + h[Process Plaintext] + i[Handle Error] + a -- success (content = PlainText) --> b +b -- success (content = PlainText) --> c +c -- success (content = ZTDF) --> d +c -- failure --> e +f -- success (content = ZTDF) --> g +g -- success (content = PlainText) --> h +g -- failure --> i \ No newline at end of file diff --git a/docs/diagrams/generic_ztdf_nifi_flows.svg b/docs/diagrams/generic_ztdf_nifi_flows.svg new file mode 100644 index 0000000..bcfcf43 --- /dev/null +++ b/docs/diagrams/generic_ztdf_nifi_flows.svg @@ -0,0 +1,4 @@ +
success (content = PlainText)
success (content = PlainText)
success (content = ZTDF)
failure
success (content = ZTDF)
success (content = PlainText)
failure
Nifi Processor
+

UpdateAttribute Add data policy attributes to FlowFile

+

ConvertToZTDF

Process ZTDF
Handle Error
Nifi Processor
+

ConvertFromZTDF

Process Plaintext
Handle Error
Generic ZTDF NiFi Flows \ No newline at end of file diff --git a/nifi-tdf-processors/src/main/java/io/opentdf/nifi/AbstractTDFProcessor.java b/nifi-tdf-processors/src/main/java/io/opentdf/nifi/AbstractTDFProcessor.java index 2ec3ba9..6536ac2 100644 --- a/nifi-tdf-processors/src/main/java/io/opentdf/nifi/AbstractTDFProcessor.java +++ b/nifi-tdf-processors/src/main/java/io/opentdf/nifi/AbstractTDFProcessor.java @@ -31,10 +31,6 @@ */ public abstract class AbstractTDFProcessor extends AbstractProcessor { - static{ - Security.addProvider(new BouncyCastleProvider()); - } - public static final PropertyDescriptor FLOWFILE_PULL_SIZE = new org.apache.nifi.components.PropertyDescriptor.Builder() .name("FlowFile queue pull limit") .description("FlowFile queue pull size limit") diff --git a/nifi-tdf-processors/src/test/java/io/opentdf/nifi/ExampleProvision.java b/nifi-tdf-processors/src/test/java/io/opentdf/nifi/ExampleProvision.java new file mode 100644 index 0000000..d6b7327 --- /dev/null +++ b/nifi-tdf-processors/src/test/java/io/opentdf/nifi/ExampleProvision.java @@ -0,0 +1,81 @@ +package io.opentdf.nifi; + +import com.google.common.util.concurrent.ListenableFuture; +import io.opentdf.platform.policy.*; +import io.opentdf.platform.policy.attributes.CreateAttributeRequest; +import io.opentdf.platform.policy.attributes.CreateAttributeResponse; +import io.opentdf.platform.policy.attributes.ListAttributesRequest; +import io.opentdf.platform.policy.namespaces.CreateNamespaceRequest; +import io.opentdf.platform.policy.namespaces.CreateNamespaceResponse; +import io.opentdf.platform.policy.namespaces.ListNamespacesRequest; +import io.opentdf.platform.policy.namespaces.ListNamespacesResponse; +import io.opentdf.platform.policy.subjectmapping.CreateSubjectMappingRequest; +import io.opentdf.platform.policy.subjectmapping.CreateSubjectMappingResponse; +import io.opentdf.platform.policy.subjectmapping.SubjectConditionSetCreate; +import io.opentdf.platform.sdk.SDK; +import io.opentdf.platform.sdk.SDKBuilder; + +import java.util.Optional; + +public class ExampleProvision { + static String clientSecret = System.getenv("CLIENT_SECRET"); + static String platformEndpoint = System.getenv("PLATFORM_ENDPOINT"); + static String NS = "opentdf.io"; + static String ATTR_NAME = "intellectualproperty"; + + public static void main(String[] args) { + try { + SDK sdk = SDKBuilder.newBuilder().platformEndpoint(platformEndpoint).build(); + ListenableFuture resp = sdk.getServices().namespaces().listNamespaces(ListNamespacesRequest.newBuilder().build()); + Optional nsOpt = resp.get().getNamespacesList().stream().filter(x -> x.getName().equals(NS)).findFirst(); + Namespace namespace = nsOpt.isPresent() ? nsOpt.get() : null; + if (namespace == null) { + ListenableFuture createNSRespFuture = sdk.getServices().namespaces().createNamespace(CreateNamespaceRequest.newBuilder().setName(NS).build()); + CreateNamespaceResponse createNamespaceResponse = createNSRespFuture.get(); + namespace = createNamespaceResponse.getNamespace(); + System.out.println("Created namespace " + NS + " " + namespace.getId()); + } else { + System.out.println(NS + " already exists"); + } + Optional attributeOptional = sdk.getServices().attributes().listAttributes(ListAttributesRequest.newBuilder(). + build()).get().getAttributesList().stream().filter(x -> x.getName().equals(ATTR_NAME)).findFirst(); + Attribute attribute = null; + if (attributeOptional.isPresent()) { + System.out.println("Attribute Exists:" + attributeOptional.get().getId()); + attribute = attributeOptional.get(); + + } else { + + ListenableFuture caRespFuture = sdk.getServices().attributes().createAttribute(CreateAttributeRequest.newBuilder() + .setName(ATTR_NAME) + .setRule(AttributeRuleTypeEnum.ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY) + .addValues("tradesecret").addValues("confidential") + .setNamespaceId(namespace.getId()).build()); + CreateAttributeResponse caResp = caRespFuture.get(); + System.out.println("Created attribute and values for " + caResp.getAttribute().getFqn() + ", id = " + caResp.getAttribute().getId()); + attribute = caResp.getAttribute(); + } + Value tradeSecret = attribute.getValuesList().stream().filter(x -> x.getValue().equals("tradesecret")).findFirst().get(); + CreateSubjectMappingResponse createSubjectMappingResponse = sdk.getServices().subjectMappings().createSubjectMapping(CreateSubjectMappingRequest.newBuilder() + .addActions(Action.newBuilder().setStandard(Action.StandardAction.STANDARD_ACTION_DECRYPT)) + .setAttributeValueId(tradeSecret.getId()).setNewSubjectConditionSet(SubjectConditionSetCreate.newBuilder() + .addSubjectSets(SubjectSet.newBuilder().addConditionGroups(ConditionGroup.newBuilder() + .setBooleanOperator(ConditionBooleanTypeEnum.CONDITION_BOOLEAN_TYPE_ENUM_AND) + .addConditions( + Condition.newBuilder().setOperator(SubjectMappingOperatorEnum.SUBJECT_MAPPING_OPERATOR_ENUM_IN) + .setSubjectExternalSelectorValue(".client_id").addSubjectExternalValues("opentdf"))))) + .build()).get(); + System.out.println("Done"); + ListenableFuture caRespFuture = sdk.getServices().attributes().createAttribute(CreateAttributeRequest.newBuilder() + .setName("intellectualproperty") + .setRule(AttributeRuleTypeEnum.ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY) + .addValues("tradesecret").addValues("confidential") + .setNamespaceId(namespace.getId()).build()); + CreateAttributeResponse caResp = caRespFuture.get(); + System.out.println("Created attribute and values for " + caResp.getAttribute().getFqn() + ", id = " + caResp.getAttribute().getId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} \ No newline at end of file diff --git a/nifi-tdf-processors/src/test/java/io/opentdf/nifi/NifiIT.java b/nifi-tdf-processors/src/test/java/io/opentdf/nifi/NifiIT.java deleted file mode 100644 index ba74b78..0000000 --- a/nifi-tdf-processors/src/test/java/io/opentdf/nifi/NifiIT.java +++ /dev/null @@ -1,46 +0,0 @@ -package io.opentdf.nifi; - -import com.google.common.util.concurrent.ListenableFuture; -import com.google.protobuf.DescriptorProtos; -import io.opentdf.platform.policy.AttributeRuleTypeEnum; -import io.opentdf.platform.policy.Namespace; -import io.opentdf.platform.policy.attributes.CreateAttributeRequest; -import io.opentdf.platform.policy.attributes.CreateAttributeResponse; -import io.opentdf.platform.policy.attributes.CreateAttributeValueRequest; -import io.opentdf.platform.policy.namespaces.*; -import io.opentdf.platform.sdk.SDK; -import io.opentdf.platform.sdk.SDKBuilder; -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class NifiIT { - - String clientId = System.getenv("CLIENT_ID"); - String clientSecret = System.getenv("CLIENT_SECRET"); - String platformEndpoint = System.getenv("PLATFORM_ENDPOINT"); - String NS="opentdf.io"; - @Test - public void testIt() throws Exception{ - SDK sdk = SDKBuilder.newBuilder().platformEndpoint(platformEndpoint) - .clientSecret(clientId, clientSecret).build(); - ListenableFuture resp = sdk.getServices().namespaces().listNamespaces(ListNamespacesRequest.newBuilder().build()); - Optional nsOpt = resp.get().getNamespacesList().stream().filter(x->x.getName().equals(NS)).findFirst(); - Namespace namespace = nsOpt.isPresent() ? nsOpt.get() : null; - if (namespace==null){ - ListenableFuture createNSRespFuture = sdk.getServices().namespaces().createNamespace(CreateNamespaceRequest.newBuilder().setName(NS).build()); - CreateNamespaceResponse createNamespaceResponse = createNSRespFuture.get(); - namespace = createNamespaceResponse.getNamespace(); - System.out.println("Created namespace " + NS + " " + namespace.getId()); - } - ListenableFuture caRespFuture = sdk.getServices().attributes().createAttribute(CreateAttributeRequest.newBuilder() - .setName("intellectualproperty") - .setRule(AttributeRuleTypeEnum.ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY) - .addValues("tradesecret").addValues("confidential") - .setNamespaceId(namespace.getId()).build()); - CreateAttributeResponse caResp = caRespFuture.get(); - System.out.println("Created attribute and values for " + caResp.getAttribute().getFqn() +", id = " + caResp.getAttribute().getId()); - } -} diff --git a/pom.xml b/pom.xml index b8bd7f0..7f12728 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ io.opentdf.platform sdk - 0.1.0-SNAPSHOT + 0.1.0 org.apache.commons