From 7e8d9ec6b67550a68a6fc7bbc6cc6687abfed851 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 17 Oct 2023 13:34:20 -0400 Subject: [PATCH] PHL-306: Renaming 'filter profile' to 'policy'. --- README.md | 18 +- .../configuration/PhileasConfiguration.java | 8 +- .../services/PhileasFilterService.java | 700 +++++++++--------- .../filters/ai/opennlp/PersonsV2Filter.java | 8 +- .../filters/ai/opennlp/PersonsV3Filter.java | 8 +- .../filters/ai/python/PersonsV1Filter.java | 14 +- .../custom/PhoneNumberRulesFilter.java | 8 +- .../services/filters/regex/AgeFilter.java | 6 +- .../regex/BankRoutingNumberFilter.java | 6 +- .../filters/regex/BitcoinAddressFilter.java | 12 +- .../filters/regex/CreditCardFilter.java | 6 +- .../filters/regex/CurrencyFilter.java | 6 +- .../services/filters/regex/DateFilter.java | 6 +- .../filters/regex/DriversLicenseFilter.java | 6 +- .../filters/regex/EmailAddressFilter.java | 6 +- .../filters/regex/IbanCodeFilter.java | 6 +- .../filters/regex/IdentifierFilter.java | 10 +- .../filters/regex/IpAddressFilter.java | 12 +- .../filters/regex/MacAddressFilter.java | 12 +- .../filters/regex/PassportNumberFilter.java | 6 +- .../regex/PhoneNumberExtensionFilter.java | 6 +- .../filters/regex/PhysicianNameFilter.java | 14 +- .../services/filters/regex/SectionFilter.java | 6 +- .../services/filters/regex/SsnFilter.java | 6 +- .../regex/StateAbbreviationFilter.java | 6 +- .../filters/regex/StreetAddressFilter.java | 6 +- .../filters/regex/TrackingNumberFilter.java | 6 +- .../services/filters/regex/UrlFilter.java | 6 +- .../services/filters/regex/VinFilter.java | 6 +- .../services/filters/regex/ZipCodeFilter.java | 6 +- .../test/phileas/services/EndToEndTests.java | 80 +- .../phileas/services/EndToEndTestsHelper.java | 85 ++- .../services/PhileasFilterServiceTest.java | 30 +- .../services/filters/AbstractFilterTest.java | 16 +- .../services/filters/AgeFilterTest.java | 46 +- .../filters/BankRoutingNumberFilterTest.java | 6 +- .../filters/BitcoinAddressFilterTest.java | 6 +- .../BloomFilterDictionaryFilterTest.java | 10 +- .../services/filters/CityFilterTest.java | 10 +- .../services/filters/CountyFilterTest.java | 6 +- .../filters/CreditCardFilterTest.java | 44 +- .../services/filters/CurrencyFilterTest.java | 14 +- .../filters/CustomDictionaryFilterTest.java | 4 +- .../services/filters/DateFilterTest.java | 94 +-- .../filters/DriversLicenseFilterTest.java | 2 +- .../filters/EmailAddressFilterTest.java | 2 +- .../phileas/services/filters/FilterTest.java | 6 +- .../services/filters/FirstNameFilterTest.java | 20 +- .../HospitalAbbreviationFilterTest.java | 2 +- .../services/filters/HospitalFilterTest.java | 2 +- .../services/filters/IbanCodeFilterTest.java | 8 +- .../filters/IdentifierFilterTest.java | 34 +- .../services/filters/IpAddressFilterTest.java | 8 +- .../filters/MacAddressFilterTest.java | 4 +- .../filters/PassportNumberFilterTest.java | 2 +- .../services/filters/PersonsV2FilterTest.java | 12 +- .../services/filters/PersonsV3FilterTest.java | 4 +- .../PhoneNumberExtensionFilterTest.java | 4 +- .../filters/PhoneNumberFilterTest.java | 16 +- .../filters/PhysicianNameFilterTest.java | 22 +- .../services/filters/SectionFilterTest.java | 6 +- .../services/filters/SsnFilterTest.java | 14 +- .../filters/StateAbbreviationFilterTest.java | 6 +- .../services/filters/StateFilterTest.java | 6 +- .../filters/StreetAddressFilterTest.java | 48 +- .../services/filters/SurnameFilterTest.java | 12 +- .../filters/TrackingNumberFilterTest.java | 24 +- .../services/filters/UrlFilterTest.java | 30 +- .../services/filters/VinFilterTest.java | 10 +- .../services/filters/ZipCodeFilterTest.java | 20 +- .../IgnoredPatternsFilterTest.java | 10 +- .../postfilters/IgnoredTermsFilterTest.java | 6 +- ...viceTest.java => S3PolicyServiceTest.java} | 66 +- .../philterd/phileas/model/domain/Domain.java | 8 +- ...ption.java => InvalidPolicyException.java} | 6 +- .../philterd/phileas/model/filter/Filter.java | 80 +- .../model/filter/rules/RulesFilter.java | 24 +- .../BloomFilterDictionaryFilter.java | 7 +- .../dictionary/LuceneDictionaryFilter.java | 21 +- .../philterd/phileas/model/objects/Alert.java | 16 +- ...rofileResult.java => GetPolicyResult.java} | 12 +- .../phileas/model/profile/Identifiers.java | 4 +- .../{FilterProfile.java => Policy.java} | 2 +- .../model/profile/filters/Identifier.java | 2 +- ...ervice.java => AbstractPolicyService.java} | 2 +- .../phileas/model/services/AlertService.java | 4 +- .../model/services/DocumentProcessor.java | 4 +- .../phileas/model/services/FilterService.java | 14 +- ...heService.java => PolicyCacheService.java} | 28 +- ...ProfileService.java => PolicyService.java} | 30 +- ...FilterProfileTest.java => PolicyTest.java} | 32 +- .../fhir/FhirDocumentProcessor.java | 10 +- .../fhir/FhirDocumentProcessorTest.java | 14 +- .../UnstructuredDocumentProcessor.java | 6 +- .../services/alerts/LocalAlertService.java | 4 +- .../services/alerts/RedisAlertService.java | 4 +- .../ai/philterd/services/pdf/PdfRedacter.java | 11 +- .../java/ai/philterd/PdfRedacterTest.java | 30 +- .../profiles/LocalFilterProfileService.java | 180 ----- .../services/profiles/LocalPolicyService.java | 180 +++++ ...ofileService.java => S3PolicyService.java} | 116 +-- .../profiles/StaticFilterProfileService.java | 110 --- .../profiles/StaticPolicyService.java | 110 +++ ...e.java => InMemoryPolicyCacheService.java} | 18 +- ...ry.java => PolicyCacheServiceFactory.java} | 12 +- ...vice.java => RedisPolicyCacheService.java} | 28 +- .../profiles/utils/FilterProfileUtils.java | 105 --- .../services/profiles/utils/PolicyUtils.java | 105 +++ .../utils/FilterProfileUtilsTest.java | 229 ------ .../profiles/utils/PolicyUtilsTest.java | 229 ++++++ 110 files changed, 1778 insertions(+), 1808 deletions(-) rename phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/{S3FilterProfileServiceTest.java => S3PolicyServiceTest.java} (73%) rename phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/{InvalidFilterProfileException.java => InvalidPolicyException.java} (79%) rename phileas-model/src/main/java/ai/philterd/phileas/model/objects/{GetFilterProfileResult.java => GetPolicyResult.java} (74%) rename phileas-model/src/main/java/ai/philterd/phileas/model/profile/{FilterProfile.java => Policy.java} (99%) rename phileas-model/src/main/java/ai/philterd/phileas/model/services/{AbstractFilterProfileService.java => AbstractPolicyService.java} (93%) rename phileas-model/src/main/java/ai/philterd/phileas/model/services/{FilterProfileCacheService.java => PolicyCacheService.java} (60%) rename phileas-model/src/main/java/ai/philterd/phileas/model/services/{FilterProfileService.java => PolicyService.java} (59%) rename phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/{FilterProfileTest.java => PolicyTest.java} (93%) delete mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalFilterProfileService.java create mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalPolicyService.java rename phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/{S3FilterProfileService.java => S3PolicyService.java} (64%) delete mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticFilterProfileService.java create mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticPolicyService.java rename phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/{InMemoryFilterProfileCacheService.java => InMemoryPolicyCacheService.java} (70%) rename phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/{FilterProfileCacheServiceFactory.java => PolicyCacheServiceFactory.java} (68%) rename phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/{RedisFilterProfileCacheService.java => RedisPolicyCacheService.java} (72%) delete mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/FilterProfileUtils.java create mode 100644 phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/PolicyUtils.java delete mode 100644 phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/FilterProfileUtilsTest.java create mode 100644 phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/PolicyUtilsTest.java diff --git a/README.md b/README.md index aa2e7e7d..e3348a47 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Phileas is the underlying core of [Philter](https://philterd.ai/philter/). * Phileas can disambiguate types of sensitive information (i.e. SSN vs. phone number). * Phileas can deidentify text consistently ("John Smith" is replaced consistently in certain documents). * Phileas can shift dates or replace dates with approximate representations (i.e. "3 months ago"). -* Phileas is customizable using "filter profiles" that define what sensitive information to find and how to redact it. +* Phileas uses policies to define what sensitive information to find and how to redact it. ## Supported PII, PHI, and Other Sensitive Information @@ -108,10 +108,10 @@ PhileasConfiguration phileasConfiguration = ConfigFactory.create(PhileasConfigur FilterService filterService = new PhileasFilterService(phileasConfiguration); -FilterResponse response = filterService.filter(filterProfiles, context, documentId, body, MimeType.TEXT_PLAIN); +FilterResponse response = filterService.filter(policies, context, documentId, body, MimeType.TEXT_PLAIN); ``` -The `filterProfiles` is a list of `FilterProfile` classes. (See below for more about Filter Profiles.) The `context` and `documentId` are arbitrary values you can use to uniquely identify the text being filtered. The `body` is the text you are filtering. Lastly, we specify that the data is plain text. +The `policies` is a list of `Policy` classes. (See below for more about Policies.) The `context` and `documentId` are arbitrary values you can use to uniquely identify the text being filtered. The `body` is the text you are filtering. Lastly, we specify that the data is plain text. The `response` contains information about the identified sensitive information along with the filtered text. @@ -128,20 +128,20 @@ PhileasConfiguration phileasConfiguration = ConfigFactory.create(PhileasConfigur FilterService filterService = new PhileasFilterService(phileasConfiguration); -BinaryDocumentFilterResponse response = filterService.filter(filterProfiles, context, documentId, body, MimeType.APPLICATION_PDF, MimeType.IMAGE_JPEG); +BinaryDocumentFilterResponse response = filterService.filter(policies, context, documentId, body, MimeType.APPLICATION_PDF, MimeType.IMAGE_JPEG); ``` -The `filterProfiles` is a list of `FilterProfile` classes which are created by deserializing a filter profile from JSON. (See below for more about Filter Profiles.) The `context` and `documentId` are arbitrary values you can use to uniquely identify the text being filtered. The `body` is the text you are filtering. Lastly, we specify that the data is plain text. +The `policies` is a list of `Policy` classes which are created by deserializing a policy from JSON. (See below for more about Filter Profiles.) The `context` and `documentId` are arbitrary values you can use to uniquely identify the text being filtered. The `body` is the text you are filtering. Lastly, we specify that the data is plain text. The `response` contains a zip file of the images generated by redacting the PDF document. -### Filter Profiles +### Policies -A "filter profile" is an instance of a `FilterProfile` class that tells Phileas the types of sensitive information to identify, and what to do with the sensitive information when found. A filter profile describes the entire filtering process, from what filters to apply, terms to ignore, to everything in between. Phileas can apply one or more filter profiles when `filter()` is called. The list of filter profiles will be applied in order as they were added to the list. +A policy is an instance of a `Policy` class that tells Phileas the types of sensitive information to identify, and what to do with the sensitive information when found. A policy describes the entire filtering process, from what filters to apply, terms to ignore, to everything in between. Phileas can apply one or more policies when `filter()` is called. The list of policies will be applied in order as they were added to the list. -For examples on creating a filter profile, look at [EndToEndTestsHelper](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/EndToEndTestsHelper.java). The [PhileasFilterServiceTest](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/PhileasFilterServiceTest.java) and [EndToEndTests](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/EndToEndTests.java) test classes have examples of how to configure Phileas and filter text. +For examples on creating a policy, look at [EndToEndTestsHelper](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/EndToEndTestsHelper.java). The [PhileasFilterServiceTest](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/PhileasFilterServiceTest.java) and [EndToEndTests](https://github.com/philterd/phileas/blob/main/phileas-core/src/test/java/io/philterd/test/phileas/services/EndToEndTests.java) test classes have examples of how to configure Phileas and filter text. -Filter profiles can be de/serialized to JSON. Here is a basic (but valid) filter profile that identifies and redacts ages: +Filter profiles can be de/serialized to JSON. Here is a basic (but valid) policy that identifies and redacts ages: ``` { diff --git a/phileas-configuration/src/main/java/ai/philterd/phileas/configuration/PhileasConfiguration.java b/phileas-configuration/src/main/java/ai/philterd/phileas/configuration/PhileasConfiguration.java index 6719ef94..e4c8635e 100644 --- a/phileas-configuration/src/main/java/ai/philterd/phileas/configuration/PhileasConfiguration.java +++ b/phileas-configuration/src/main/java/ai/philterd/phileas/configuration/PhileasConfiguration.java @@ -119,18 +119,18 @@ public interface PhileasConfiguration extends Config { // Filter Profiles @DefaultValue("./profiles/") @Key("filter.profiles.directory") - String filterProfilesDirectory(); + String policiesDirectory(); @Key("filter.profiles.s3.bucket") - String filterProfilesS3Bucket(); + String policiesS3Bucket(); @DefaultValue("") @Key("filter.profiles.s3.prefix") - String filterProfilesS3Prefix(); + String policiesS3Prefix(); @DefaultValue("us-east-1") @Key("filter.profiles.s3.region") - String filterProfilesS3Region(); + String policiesS3Region(); // Metrics @DefaultValue("philter") diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java b/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java index d467d472..07f710aa 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java @@ -32,7 +32,7 @@ import ai.philterd.phileas.model.filter.rules.dictionary.BloomFilterDictionaryFilter; import ai.philterd.phileas.model.filter.rules.dictionary.LuceneDictionaryFilter; import ai.philterd.phileas.model.objects.*; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Ignored; import ai.philterd.phileas.model.profile.filters.CustomDictionary; import ai.philterd.phileas.model.profile.filters.Identifier; @@ -52,9 +52,9 @@ import ai.philterd.phileas.services.filters.custom.PhoneNumberRulesFilter; import ai.philterd.phileas.services.filters.regex.*; import ai.philterd.phileas.services.postfilters.*; -import ai.philterd.phileas.services.profiles.LocalFilterProfileService; -import ai.philterd.phileas.services.profiles.S3FilterProfileService; -import ai.philterd.phileas.services.profiles.utils.FilterProfileUtils; +import ai.philterd.phileas.services.profiles.LocalPolicyService; +import ai.philterd.phileas.services.profiles.S3PolicyService; +import ai.philterd.phileas.services.profiles.utils.PolicyUtils; import ai.philterd.phileas.services.split.SplitFactory; import ai.philterd.phileas.services.validators.DateSpanValidator; import ai.philterd.services.pdf.PdfRedacter; @@ -79,8 +79,8 @@ public class PhileasFilterService implements FilterService { private final PhileasConfiguration phileasConfiguration; - private final FilterProfileService filterProfileService; - private final FilterProfileUtils filterProfileUtils; + private final PolicyService policyService; + private final PolicyUtils policyUtils; private final MetricsService metricsService; private final Map stats; @@ -117,9 +117,9 @@ public PhileasFilterService(PhileasConfiguration phileasConfiguration) throws IO // Configure metrics. this.metricsService = new PhileasMetricsService(phileasConfiguration); - // Set the filter profile services. - this.filterProfileService = buildFilterProfileService(phileasConfiguration); - this.filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); + // Set the policy services. + this.policyService = buildPolicyService(phileasConfiguration); + this.policyUtils = new PolicyUtils(policyService, gson); // Set the anonymization cache service. this.anonymizationCacheService = AnonymizationCacheServiceFactory.getAnonymizationCacheService(phileasConfiguration); @@ -152,8 +152,8 @@ public PhileasFilterService(PhileasConfiguration phileasConfiguration) throws IO } @Override - public FilterProfileService getFilterProfileService() { - return filterProfileService; + public PolicyService getPolicyService() { + return policyService; } @Override @@ -162,23 +162,23 @@ public AlertService getAlertService() { } @Override - public FilterResponse filter(List filterProfileNames, String context, String documentId, String input, MimeType mimeType) throws Exception { + public FilterResponse filter(List policyNames, String context, String documentId, String input, MimeType mimeType) throws Exception { - // Get the filter profile. - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(filterProfileNames); + // Get the policy. + final Policy policy = policyUtils.getCombinedPolicys(policyNames); // Load default values based on the domain. - if(StringUtils.equalsIgnoreCase(Domain.DOMAIN_LEGAL, filterProfile.getDomain())) { + if(StringUtils.equalsIgnoreCase(Domain.DOMAIN_LEGAL, policy.getDomain())) { // PHL-209: Implement legal domain. - filterProfile.getIgnored().add(LegalDomain.getInstance().getIgnored()); + policy.getIgnored().add(LegalDomain.getInstance().getIgnored()); // TODO: Add filters. - } else if(StringUtils.equalsIgnoreCase(Domain.DOMAIN_HEALTH, filterProfile.getDomain())) { + } else if(StringUtils.equalsIgnoreCase(Domain.DOMAIN_HEALTH, policy.getDomain())) { // PHL-210: Implement health domain. - filterProfile.getIgnored().add(HealthDomain.getInstance().getIgnored()); + policy.getIgnored().add(HealthDomain.getInstance().getIgnored()); // TODO: Add filters. @@ -186,20 +186,20 @@ public FilterResponse filter(List filterProfileNames, String context, St // Analyze the document. final DocumentAnalysis documentAnalysis; - if(filterProfile.getConfig().getAnalysis().isEnabled()) { + if(policy.getConfig().getAnalysis().isEnabled()) { documentAnalysis = documentAnalyzer.analyze(input); } else { documentAnalysis = new DocumentAnalysis(); } - final List filters = getFiltersForFilterProfile(filterProfile, documentAnalysis); - final List postFilters = getPostFiltersForFilterProfile(filterProfile); + final List filters = getFiltersForPolicy(policy, documentAnalysis); + final List postFilters = getPostFiltersForPolicy(policy); // See if we need to generate a document ID. if(StringUtils.isEmpty(documentId)) { // PHL-58: Use a hash function to generate the document ID. - documentId = DigestUtils.md5Hex(UUID.randomUUID().toString() + "-" + context + "-" + filterProfile.getName() + "-" + input); + documentId = DigestUtils.md5Hex(UUID.randomUUID().toString() + "-" + context + "-" + policy.getName() + "-" + input); LOGGER.debug("Generated document ID {}", documentId); } @@ -209,10 +209,10 @@ public FilterResponse filter(List filterProfileNames, String context, St if(mimeType == MimeType.TEXT_PLAIN) { // PHL-145: Do we need to split the input text due to its size? - if (filterProfile.getConfig().getSplitting().isEnabled() && input.length() >= filterProfile.getConfig().getSplitting().getThreshold()) { + if (policy.getConfig().getSplitting().isEnabled() && input.length() >= policy.getConfig().getSplitting().getThreshold()) { - // Get the splitter to use from the filter profile. - final SplitService splitService = SplitFactory.getSplitService(filterProfile.getConfig().getSplitting().getMethod()); + // Get the splitter to use from the policy. + final SplitService splitService = SplitFactory.getSplitService(policy.getConfig().getSplitting().getMethod()); // Holds all of the filter responses that will ultimately be combined into a single response. final List filterResponses = new LinkedList<>(); @@ -222,7 +222,7 @@ public FilterResponse filter(List filterProfileNames, String context, St // Process each split. for (int i = 0; i < splits.size(); i++) { - filterResponses.add(unstructuredDocumentProcessor.process(filterProfile, filters, postFilters, context, documentId, i, splits.get(i))); + filterResponses.add(unstructuredDocumentProcessor.process(policy, filters, postFilters, context, documentId, i, splits.get(i))); } // Combine the results into a single filterResponse object. @@ -231,7 +231,7 @@ public FilterResponse filter(List filterProfileNames, String context, St } else { // Do not split. Process the entire string at once. - filterResponse = unstructuredDocumentProcessor.process(filterProfile, filters, postFilters, context, documentId, 0, input); + filterResponse = unstructuredDocumentProcessor.process(policy, filters, postFilters, context, documentId, 0, input); } @@ -245,16 +245,16 @@ public FilterResponse filter(List filterProfileNames, String context, St } @Override - public BinaryDocumentFilterResponse filter(List filterProfileNames, String context, String documentId, byte[] input, MimeType mimeType, MimeType outputMimeType) throws Exception { + public BinaryDocumentFilterResponse filter(List policyNames, String context, String documentId, byte[] input, MimeType mimeType, MimeType outputMimeType) throws Exception { - // Get the filter profile. - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(filterProfileNames); + // Get the policy. + final Policy policy = policyUtils.getCombinedPolicys(policyNames); // See if we need to generate a document ID. if(StringUtils.isEmpty(documentId)) { // PHL-58: Use a hash function to generate the document ID. - documentId = DigestUtils.md5Hex(UUID.randomUUID().toString() + "-" + context + "-" + filterProfile.getName() + "-" + input); + documentId = DigestUtils.md5Hex(UUID.randomUUID().toString() + "-" + context + "-" + policy.getName() + "-" + input); LOGGER.debug("Generated document ID {}", documentId); } @@ -280,17 +280,17 @@ public BinaryDocumentFilterResponse filter(List filterProfileNames, Stri // Analyze the lines to determine the type of document. final DocumentAnalysis documentAnalysis; - if (filterProfile.getConfig().getAnalysis().isEnabled()) { + if (policy.getConfig().getAnalysis().isEnabled()) { documentAnalysis = documentAnalyzer.analyze(lines); } else { documentAnalysis = new DocumentAnalysis(); } - final List filters = getFiltersForFilterProfile(filterProfile, documentAnalysis); - final List postFilters = getPostFiltersForFilterProfile(filterProfile); + final List filters = getFiltersForPolicy(policy, documentAnalysis); + final List postFilters = getPostFiltersForPolicy(policy); // TODO: The following code really only needs to be done if there is at least - // one filter defined in the filter profile. + // one filter defined in the policy. // Process each line looking for sensitive information in each line. for (final String line : lines) { @@ -298,7 +298,7 @@ public BinaryDocumentFilterResponse filter(List filterProfileNames, Stri final int piece = 0; // Process the text. - final FilterResponse filterResponse = unstructuredDocumentProcessor.process(filterProfile, filters, postFilters, context, documentId, piece, line); + final FilterResponse filterResponse = unstructuredDocumentProcessor.process(policy, filters, postFilters, context, documentId, piece, line); // Add all the found spans to the list of spans. spans.addAll(filterResponse.getExplanation().getAppliedSpans()); @@ -313,15 +313,15 @@ public BinaryDocumentFilterResponse filter(List filterProfileNames, Stri } - // TODO: Build this from the config in the filter profile. + // TODO: Build this from the config in the policy. final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); pdfRedactionOptions.setDpi(150); pdfRedactionOptions.setScale(0.25f); pdfRedactionOptions.setCompressionQuality(1.0f); - // Redact those terms in the document along with any bounding boxes identified in the filter profile. - final List boundingBoxes = getBoundingBoxes(filterProfile, mimeType); - final Redacter redacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + // Redact those terms in the document along with any bounding boxes identified in the policy. + final List boundingBoxes = getBoundingBoxes(policy, mimeType); + final Redacter redacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = redacter.process(input, outputMimeType); // Create the response. @@ -341,7 +341,7 @@ public BinaryDocumentFilterResponse filter(List filterProfileNames, Stri /*} else if(mimeType == MimeType.IMAGE_JPEG) { // PHL-223: Face recognition - // TODO: Get options from the filter profile. + // TODO: Get options from the policy. final ImageFilterResponse imageFilterResponse = imageProcessor.process(input); // TODO: Explanation? @@ -360,16 +360,16 @@ public BinaryDocumentFilterResponse filter(List filterProfileNames, Stri } /** - * Get the bounding boxes from the filter profile for a given mime type. - * @param filterProfile The filter profile. + * Get the bounding boxes from the policy for a given mime type. + * @param policy The policy. * @param mimeType The mime type. - * @return A list of bounding boxes from the filter profile for the given mime type. + * @return A list of bounding boxes from the policy for the given mime type. */ - private List getBoundingBoxes(final FilterProfile filterProfile, final MimeType mimeType) { + private List getBoundingBoxes(final Policy policy, final MimeType mimeType) { final List boundingBoxes = new LinkedList<>(); - for(final BoundingBox boundingBox : filterProfile.getGraphical().getBoundingBoxes()) { + for(final BoundingBox boundingBox : policy.getGraphical().getBoundingBoxes()) { if(StringUtils.equalsIgnoreCase(boundingBox.getMimeType(), mimeType.toString())) { boundingBoxes.add(boundingBox); } @@ -379,31 +379,31 @@ private List getBoundingBoxes(final FilterProfile filterProfile, fi } - private FilterProfileService buildFilterProfileService(PhileasConfiguration phileasConfiguration) throws IOException { + private PolicyService buildPolicyService(PhileasConfiguration phileasConfiguration) throws IOException { - final FilterProfileService filterProfileService; - final String s3Bucket = phileasConfiguration.filterProfilesS3Bucket(); + final PolicyService policyService; + final String s3Bucket = phileasConfiguration.policiesS3Bucket(); - // If an S3 bucket is provided then instantiate an S3FilterProfileService. + // If an S3 bucket is provided then instantiate an S3PolicyService. if(StringUtils.isNotEmpty(s3Bucket)) { - LOGGER.info("Initializing configuration for filter profiles S3 bucket."); - filterProfileService = new S3FilterProfileService(phileasConfiguration, false); + LOGGER.info("Initializing configuration for policies S3 bucket."); + policyService = new S3PolicyService(phileasConfiguration, false); } else { - LOGGER.info("Using local storage for filter profiles."); - filterProfileService = new LocalFilterProfileService(phileasConfiguration); + LOGGER.info("Using local storage for policies."); + policyService = new LocalPolicyService(phileasConfiguration); } - return filterProfileService; + return policyService; } - private List getPostFiltersForFilterProfile(final FilterProfile filterProfile) throws IOException { + private List getPostFiltersForPolicy(final Policy policy) throws IOException { - LOGGER.debug("Reloading filter profiles."); + LOGGER.debug("Reloading policies."); final List postFilters = new LinkedList<>(); @@ -419,10 +419,10 @@ private List getPostFiltersForFilterProfile(final FilterProfile filt // Ignored terms filter. Looks for ignored terms in the scope of the whole document (and not just a particular filter). // No matter what filter found the span, it is subject to this ignore list. - if(CollectionUtils.isNotEmpty(filterProfile.getIgnored())) { + if(CollectionUtils.isNotEmpty(policy.getIgnored())) { // Make a post filter for each Ignored item in the list. - for(final Ignored ignored : filterProfile.getIgnored()) { + for(final Ignored ignored : policy.getIgnored()) { postFilters.add(new IgnoredTermsFilter(ignored)); } @@ -430,21 +430,21 @@ private List getPostFiltersForFilterProfile(final FilterProfile filt // Ignored patterns filter. Looks for terms matching a pattern in the scope of the whole document (and not just a particular filter). // No matter what filter found the span, it is subject to this ignore list. - if(CollectionUtils.isNotEmpty(filterProfile.getIgnoredPatterns())) { - postFilters.add(new IgnoredPatternsFilter(filterProfile.getIgnoredPatterns())); + if(CollectionUtils.isNotEmpty(policy.getIgnoredPatterns())) { + postFilters.add(new IgnoredPatternsFilter(policy.getIgnoredPatterns())); } - // Add the post filters if they are enabled in the filter profile. + // Add the post filters if they are enabled in the policy. - if(filterProfile.getConfig().getPostFilters().isRemoveTrailingPeriods()) { + if(policy.getConfig().getPostFilters().isRemoveTrailingPeriods()) { postFilters.add(TrailingPeriodPostFilter.getInstance()); } - if(filterProfile.getConfig().getPostFilters().isRemoveTrailingSpaces()) { + if(policy.getConfig().getPostFilters().isRemoveTrailingSpaces()) { postFilters.add(TrailingSpacePostFilter.getInstance()); } - if(filterProfile.getConfig().getPostFilters().isRemoveTrailingNewLines()) { + if(policy.getConfig().getPostFilters().isRemoveTrailingNewLines()) { postFilters.add(TrailingNewLinePostFilter.getInstance()); } @@ -452,425 +452,425 @@ private List getPostFiltersForFilterProfile(final FilterProfile filt } - private List getFiltersForFilterProfile(final FilterProfile filterProfile, DocumentAnalysis documentAnalysis) throws Exception { + private List getFiltersForPolicy(final Policy policy, DocumentAnalysis documentAnalysis) throws Exception { - LOGGER.debug("Getting filters for filter profile [{}]", filterProfile.getName()); + LOGGER.debug("Getting filters for policy [{}]", policy.getName()); // See if this filter is already cached. - filterCache.putIfAbsent(filterProfile.getName(), new ConcurrentHashMap<>()); - final Map cache = filterCache.get(filterProfile.getName()); + filterCache.putIfAbsent(policy.getName(), new ConcurrentHashMap<>()); + final Map cache = filterCache.get(policy.getName()); final List enabledFilters = new LinkedList<>(); // Rules filters. - if(filterProfile.getIdentifiers().hasFilter(FilterType.AGE) && filterProfile.getIdentifiers().getAge().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.AGE) && policy.getIdentifiers().getAge().isEnabled()) { if(cache.containsKey(FilterType.AGE)) { enabledFilters.add(cache.get(FilterType.AGE)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getAge().getAgeFilterStrategies()) + .withStrategies(policy.getIdentifiers().getAge().getAgeFilterStrategies()) .withAnonymizationService(new AgeAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getAge().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getAge().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getAge().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getAge().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getAge().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getAge().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new AgeFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.AGE, filter); + filterCache.get(policy.getName()).put(FilterType.AGE, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.BANK_ROUTING_NUMBER) && filterProfile.getIdentifiers().getBankRoutingNumber().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.BANK_ROUTING_NUMBER) && policy.getIdentifiers().getBankRoutingNumber().isEnabled()) { if(cache.containsKey(FilterType.BANK_ROUTING_NUMBER)) { enabledFilters.add(cache.get(FilterType.BANK_ROUTING_NUMBER)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getBankRoutingNumber().getBankRoutingNumberFilterStrategies()) + .withStrategies(policy.getIdentifiers().getBankRoutingNumber().getBankRoutingNumberFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getBankRoutingNumber().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getBankRoutingNumber().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getBankRoutingNumber().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getBankRoutingNumber().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getBankRoutingNumber().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getBankRoutingNumber().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new BankRoutingNumberFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.BANK_ROUTING_NUMBER, filter); + filterCache.get(policy.getName()).put(FilterType.BANK_ROUTING_NUMBER, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.BITCOIN_ADDRESS) && filterProfile.getIdentifiers().getBitcoinAddress().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.BITCOIN_ADDRESS) && policy.getIdentifiers().getBitcoinAddress().isEnabled()) { if(cache.containsKey(FilterType.BITCOIN_ADDRESS)) { enabledFilters.add(cache.get(FilterType.BITCOIN_ADDRESS)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getBitcoinAddress().getBitcoinFilterStrategies()) + .withStrategies(policy.getIdentifiers().getBitcoinAddress().getBitcoinFilterStrategies()) .withAnonymizationService(new BitcoinAddressAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getBitcoinAddress().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getBitcoinAddress().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getBitcoinAddress().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getBitcoinAddress().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getBitcoinAddress().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getBitcoinAddress().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new BitcoinAddressFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.BITCOIN_ADDRESS, filter); + filterCache.get(policy.getName()).put(FilterType.BITCOIN_ADDRESS, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.CREDIT_CARD) && filterProfile.getIdentifiers().getCreditCard().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.CREDIT_CARD) && policy.getIdentifiers().getCreditCard().isEnabled()) { if(cache.containsKey(FilterType.CREDIT_CARD)) { enabledFilters.add(cache.get(FilterType.CREDIT_CARD)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getCreditCard().getCreditCardFilterStrategies()) + .withStrategies(policy.getIdentifiers().getCreditCard().getCreditCardFilterStrategies()) .withAnonymizationService(new CreditCardAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getCreditCard().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getCreditCard().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getCreditCard().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getCreditCard().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getCreditCard().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getCreditCard().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean onlyValidCreditCardNumbers = filterProfile.getIdentifiers().getCreditCard().isOnlyValidCreditCardNumbers(); + final boolean onlyValidCreditCardNumbers = policy.getIdentifiers().getCreditCard().isOnlyValidCreditCardNumbers(); final Filter filter = new CreditCardFilter(filterConfiguration, onlyValidCreditCardNumbers); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.CREDIT_CARD, filter); + filterCache.get(policy.getName()).put(FilterType.CREDIT_CARD, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.CURRENCY) && filterProfile.getIdentifiers().getCurrency().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.CURRENCY) && policy.getIdentifiers().getCurrency().isEnabled()) { if(cache.containsKey(FilterType.CURRENCY)) { enabledFilters.add(cache.get(FilterType.CURRENCY)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getCurrency().getCurrencyFilterStrategies()) + .withStrategies(policy.getIdentifiers().getCurrency().getCurrencyFilterStrategies()) .withAnonymizationService(new CurrencyAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getCurrency().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getCurrency().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getCurrency().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getCurrency().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getCurrency().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getCurrency().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new CurrencyFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.CURRENCY, filter); + filterCache.get(policy.getName()).put(FilterType.CURRENCY, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.DATE) && filterProfile.getIdentifiers().getDate().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.DATE) && policy.getIdentifiers().getDate().isEnabled()) { if(cache.containsKey(FilterType.DATE)) { enabledFilters.add(cache.get(FilterType.DATE)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getDate().getDateFilterStrategies()) + .withStrategies(policy.getIdentifiers().getDate().getDateFilterStrategies()) .withAnonymizationService(new DateAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getDate().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getDate().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getDate().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getDate().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getDate().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getDate().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean onlyValidDates = filterProfile.getIdentifiers().getDate().isOnlyValidDates(); + final boolean onlyValidDates = policy.getIdentifiers().getDate().isOnlyValidDates(); final SpanValidator dateSpanValidator = DateSpanValidator.getInstance(); final Filter filter = new DateFilter(filterConfiguration, onlyValidDates, dateSpanValidator); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.DATE, filter); + filterCache.get(policy.getName()).put(FilterType.DATE, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.DRIVERS_LICENSE_NUMBER) && filterProfile.getIdentifiers().getDriversLicense().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.DRIVERS_LICENSE_NUMBER) && policy.getIdentifiers().getDriversLicense().isEnabled()) { if(cache.containsKey(FilterType.DRIVERS_LICENSE_NUMBER)) { enabledFilters.add(cache.get(FilterType.DRIVERS_LICENSE_NUMBER)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getDriversLicense().getDriversLicenseFilterStrategies()) + .withStrategies(policy.getIdentifiers().getDriversLicense().getDriversLicenseFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getDriversLicense().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getDriversLicense().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getDriversLicense().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getDriversLicense().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getDriversLicense().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getDriversLicense().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new DriversLicenseFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.DRIVERS_LICENSE_NUMBER, filter); + filterCache.get(policy.getName()).put(FilterType.DRIVERS_LICENSE_NUMBER, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.EMAIL_ADDRESS) && filterProfile.getIdentifiers().getEmailAddress().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.EMAIL_ADDRESS) && policy.getIdentifiers().getEmailAddress().isEnabled()) { if(cache.containsKey(FilterType.EMAIL_ADDRESS)) { enabledFilters.add(cache.get(FilterType.EMAIL_ADDRESS)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getEmailAddress().getEmailAddressFilterStrategies()) + .withStrategies(policy.getIdentifiers().getEmailAddress().getEmailAddressFilterStrategies()) .withAnonymizationService(new EmailAddressAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getEmailAddress().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getEmailAddress().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getEmailAddress().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getEmailAddress().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getEmailAddress().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getEmailAddress().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new EmailAddressFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.EMAIL_ADDRESS, filter); + filterCache.get(policy.getName()).put(FilterType.EMAIL_ADDRESS, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.IBAN_CODE) && filterProfile.getIdentifiers().getIbanCode().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.IBAN_CODE) && policy.getIdentifiers().getIbanCode().isEnabled()) { if(cache.containsKey(FilterType.IBAN_CODE)) { enabledFilters.add(cache.get(FilterType.IBAN_CODE)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getIbanCode().getIbanCodeFilterStrategies()) + .withStrategies(policy.getIdentifiers().getIbanCode().getIbanCodeFilterStrategies()) .withAnonymizationService(new IbanCodeAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getIbanCode().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getIbanCode().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getIbanCode().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getIbanCode().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getIbanCode().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getIbanCode().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean onlyValidIBANCodes = filterProfile.getIdentifiers().getIbanCode().isOnlyValidIBANCodes(); - final boolean allowSpaces = filterProfile.getIdentifiers().getIbanCode().isAllowSpaces(); + final boolean onlyValidIBANCodes = policy.getIdentifiers().getIbanCode().isOnlyValidIBANCodes(); + final boolean allowSpaces = policy.getIdentifiers().getIbanCode().isAllowSpaces(); final Filter filter = new IbanCodeFilter(filterConfiguration, onlyValidIBANCodes, allowSpaces); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.IBAN_CODE, filter); + filterCache.get(policy.getName()).put(FilterType.IBAN_CODE, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.IP_ADDRESS) && filterProfile.getIdentifiers().getIpAddress().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.IP_ADDRESS) && policy.getIdentifiers().getIpAddress().isEnabled()) { if(cache.containsKey(FilterType.IP_ADDRESS)) { enabledFilters.add(cache.get(FilterType.IP_ADDRESS)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getIpAddress().getIpAddressFilterStrategies()) + .withStrategies(policy.getIdentifiers().getIpAddress().getIpAddressFilterStrategies()) .withAnonymizationService(new IpAddressAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getIpAddress().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getIpAddress().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getIpAddress().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getIpAddress().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getIpAddress().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getIpAddress().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new IpAddressFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.IP_ADDRESS, filter); + filterCache.get(policy.getName()).put(FilterType.IP_ADDRESS, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.MAC_ADDRESS) && filterProfile.getIdentifiers().getMacAddress().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.MAC_ADDRESS) && policy.getIdentifiers().getMacAddress().isEnabled()) { if(cache.containsKey(FilterType.MAC_ADDRESS)) { enabledFilters.add(cache.get(FilterType.MAC_ADDRESS)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getMacAddress().getMacAddressFilterStrategies()) + .withStrategies(policy.getIdentifiers().getMacAddress().getMacAddressFilterStrategies()) .withAnonymizationService(new MacAddressAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getMacAddress().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getMacAddress().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getMacAddress().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getMacAddress().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getMacAddress().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getMacAddress().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new MacAddressFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.MAC_ADDRESS, filter); + filterCache.get(policy.getName()).put(FilterType.MAC_ADDRESS, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PASSPORT_NUMBER) && filterProfile.getIdentifiers().getPassportNumber().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PASSPORT_NUMBER) && policy.getIdentifiers().getPassportNumber().isEnabled()) { if(cache.containsKey(FilterType.PASSPORT_NUMBER)) { enabledFilters.add(cache.get(FilterType.PASSPORT_NUMBER)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPassportNumber().getPassportNumberFilterStrategies()) + .withStrategies(policy.getIdentifiers().getPassportNumber().getPassportNumberFilterStrategies()) .withAnonymizationService(new PassportNumberAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPassportNumber().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPassportNumber().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPassportNumber().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getPassportNumber().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPassportNumber().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPassportNumber().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PassportNumberFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PASSPORT_NUMBER, filter); + filterCache.get(policy.getName()).put(FilterType.PASSPORT_NUMBER, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PHONE_NUMBER_EXTENSION) && filterProfile.getIdentifiers().getPhoneNumberExtension().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PHONE_NUMBER_EXTENSION) && policy.getIdentifiers().getPhoneNumberExtension().isEnabled()) { if(cache.containsKey(FilterType.PHONE_NUMBER_EXTENSION)) { enabledFilters.add(cache.get(FilterType.PHONE_NUMBER_EXTENSION)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPhoneNumberExtension().getPhoneNumberExtensionFilterStrategies()) + .withStrategies(policy.getIdentifiers().getPhoneNumberExtension().getPhoneNumberExtensionFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPhoneNumberExtension().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPhoneNumberExtension().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPhoneNumberExtension().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPhoneNumberExtension().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPhoneNumberExtension().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPhoneNumberExtension().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PhoneNumberExtensionFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PHONE_NUMBER_EXTENSION, filter); + filterCache.get(policy.getName()).put(FilterType.PHONE_NUMBER_EXTENSION, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PHONE_NUMBER) && filterProfile.getIdentifiers().getPhoneNumber().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PHONE_NUMBER) && policy.getIdentifiers().getPhoneNumber().isEnabled()) { if(cache.containsKey(FilterType.PHONE_NUMBER)) { enabledFilters.add(cache.get(FilterType.PHONE_NUMBER)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPhoneNumber().getPhoneNumberFilterStrategies()) + .withStrategies(policy.getIdentifiers().getPhoneNumber().getPhoneNumberFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPhoneNumber().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPhoneNumber().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPhoneNumber().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPhoneNumber().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPhoneNumber().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPhoneNumber().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PhoneNumberRulesFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PHONE_NUMBER, filter); + filterCache.get(policy.getName()).put(FilterType.PHONE_NUMBER, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PHYSICIAN_NAME) && filterProfile.getIdentifiers().getPhysicianName().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PHYSICIAN_NAME) && policy.getIdentifiers().getPhysicianName().isEnabled()) { if(cache.containsKey(FilterType.PHYSICIAN_NAME)) { enabledFilters.add(cache.get(FilterType.PHYSICIAN_NAME)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPhysicianName().getPhysicianNameFilterStrategies()) + .withStrategies(policy.getIdentifiers().getPhysicianName().getPhysicianNameFilterStrategies()) .withAnonymizationService(new PersonsAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPhysicianName().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPhysicianName().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPhysicianName().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPhysicianName().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPhysicianName().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPhysicianName().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PhysicianNameFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PHYSICIAN_NAME, filter); + filterCache.get(policy.getName()).put(FilterType.PHYSICIAN_NAME, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.SECTION)) { + if(policy.getIdentifiers().hasFilter(FilterType.SECTION)) { - final List
sections = filterProfile.getIdentifiers().getSections(); + final List
sections = policy.getIdentifiers().getSections(); for(final Section section : sections) { @@ -883,7 +883,7 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil .withIgnored(section.getIgnored()) .withIgnoredFiles(section.getIgnoredFiles()) .withIgnoredPatterns(section.getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); @@ -899,194 +899,194 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil } - if(filterProfile.getIdentifiers().hasFilter(FilterType.SSN) && filterProfile.getIdentifiers().getSsn().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.SSN) && policy.getIdentifiers().getSsn().isEnabled()) { if(cache.containsKey(FilterType.SSN)) { enabledFilters.add(cache.get(FilterType.SSN)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getSsn().getSsnFilterStrategies()) + .withStrategies(policy.getIdentifiers().getSsn().getSsnFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getSsn().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getSsn().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getSsn().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getSsn().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getSsn().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getSsn().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new SsnFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.SSN, filter); + filterCache.get(policy.getName()).put(FilterType.SSN, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.STATE_ABBREVIATION) && filterProfile.getIdentifiers().getStateAbbreviation().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.STATE_ABBREVIATION) && policy.getIdentifiers().getStateAbbreviation().isEnabled()) { if(cache.containsKey(FilterType.STATE_ABBREVIATION)) { enabledFilters.add(cache.get(FilterType.STATE_ABBREVIATION)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getStateAbbreviation().getStateAbbreviationsFilterStrategies()) + .withStrategies(policy.getIdentifiers().getStateAbbreviation().getStateAbbreviationsFilterStrategies()) .withAnonymizationService(new StateAbbreviationAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getStateAbbreviation().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getStateAbbreviation().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getStateAbbreviation().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getStateAbbreviation().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getStateAbbreviation().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getStateAbbreviation().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new StateAbbreviationFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.STATE_ABBREVIATION, filter); + filterCache.get(policy.getName()).put(FilterType.STATE_ABBREVIATION, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.STREET_ADDRESS) && filterProfile.getIdentifiers().getStreetAddress().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.STREET_ADDRESS) && policy.getIdentifiers().getStreetAddress().isEnabled()) { if(cache.containsKey(FilterType.STREET_ADDRESS)) { enabledFilters.add(cache.get(FilterType.STREET_ADDRESS)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getStreetAddress().getStreetAddressFilterStrategies()) + .withStrategies(policy.getIdentifiers().getStreetAddress().getStreetAddressFilterStrategies()) .withAnonymizationService(new StreetAddressAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getStreetAddress().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getStreetAddress().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getStreetAddress().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getStreetAddress().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getStreetAddress().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getStreetAddress().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new StreetAddressFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.STREET_ADDRESS, filter); + filterCache.get(policy.getName()).put(FilterType.STREET_ADDRESS, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.TRACKING_NUMBER) && filterProfile.getIdentifiers().getTrackingNumber().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.TRACKING_NUMBER) && policy.getIdentifiers().getTrackingNumber().isEnabled()) { if(cache.containsKey(FilterType.TRACKING_NUMBER)) { enabledFilters.add(cache.get(FilterType.TRACKING_NUMBER)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getTrackingNumber().getTrackingNumberFilterStrategies()) + .withStrategies(policy.getIdentifiers().getTrackingNumber().getTrackingNumberFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getTrackingNumber().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getTrackingNumber().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getTrackingNumber().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getTrackingNumber().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getTrackingNumber().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getTrackingNumber().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean ups = filterProfile.getIdentifiers().getTrackingNumber().isUps(); - final boolean fedex = filterProfile.getIdentifiers().getTrackingNumber().isFedex(); - final boolean usps = filterProfile.getIdentifiers().getTrackingNumber().isUsps(); + final boolean ups = policy.getIdentifiers().getTrackingNumber().isUps(); + final boolean fedex = policy.getIdentifiers().getTrackingNumber().isFedex(); + final boolean usps = policy.getIdentifiers().getTrackingNumber().isUsps(); final Filter filter = new TrackingNumberFilter(filterConfiguration, ups, fedex, usps); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.TRACKING_NUMBER, filter); + filterCache.get(policy.getName()).put(FilterType.TRACKING_NUMBER, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.URL) && filterProfile.getIdentifiers().getUrl().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.URL) && policy.getIdentifiers().getUrl().isEnabled()) { if(cache.containsKey(FilterType.URL)) { enabledFilters.add(cache.get(FilterType.URL)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getUrl().getUrlFilterStrategies()) + .withStrategies(policy.getIdentifiers().getUrl().getUrlFilterStrategies()) .withAnonymizationService(new UrlAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getUrl().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getUrl().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getUrl().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getUrl().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getUrl().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getUrl().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean requireHttpWwwPrefix = filterProfile.getIdentifiers().getUrl().isRequireHttpWwwPrefix(); + final boolean requireHttpWwwPrefix = policy.getIdentifiers().getUrl().isRequireHttpWwwPrefix(); final Filter filter = new UrlFilter(filterConfiguration, requireHttpWwwPrefix); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.URL, filter); + filterCache.get(policy.getName()).put(FilterType.URL, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.VIN) && filterProfile.getIdentifiers().getVin().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.VIN) && policy.getIdentifiers().getVin().isEnabled()) { if(cache.containsKey(FilterType.VIN)) { enabledFilters.add(cache.get(FilterType.VIN)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getVin().getVinFilterStrategies()) + .withStrategies(policy.getIdentifiers().getVin().getVinFilterStrategies()) .withAnonymizationService(new AlphanumericAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getVin().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getVin().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getVin().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) - .withFPE(filterProfile.getFpe()) + .withIgnored(policy.getIdentifiers().getVin().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getVin().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getVin().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) + .withFPE(policy.getFpe()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new VinFilter(filterConfiguration); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.VIN, filter); + filterCache.get(policy.getName()).put(FilterType.VIN, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE) && filterProfile.getIdentifiers().getZipCode().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE) && policy.getIdentifiers().getZipCode().isEnabled()) { if(cache.containsKey(FilterType.ZIP_CODE)) { enabledFilters.add(cache.get(FilterType.ZIP_CODE)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getZipCode().getZipCodeFilterStrategies()) + .withStrategies(policy.getIdentifiers().getZipCode().getZipCodeFilterStrategies()) .withAnonymizationService(new ZipCodeAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getZipCode().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getZipCode().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getZipCode().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getZipCode().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getZipCode().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getZipCode().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final boolean requireDelimiter = filterProfile.getIdentifiers().getZipCode().isRequireDelimiter(); + final boolean requireDelimiter = policy.getIdentifiers().getZipCode().isRequireDelimiter(); final Filter filter = new ZipCodeFilter(filterConfiguration, requireDelimiter); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.ZIP_CODE, filter); + filterCache.get(policy.getName()).put(FilterType.ZIP_CODE, filter); } @@ -1094,9 +1094,9 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil // Custom dictionary filters. - if(filterProfile.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)) { + if(policy.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)) { - LOGGER.info("Filter profile {} has {} custom dictionaries.", filterProfile.getName(), filterProfile.getIdentifiers().getCustomDictionaries().size()); + LOGGER.info("Filter profile {} has {} custom dictionaries.", policy.getName(), policy.getIdentifiers().getCustomDictionaries().size()); // We keep track of the index of the custom dictionary in the list so we know // how to retrieve the strategy for the custom dictionary. This is because @@ -1105,9 +1105,9 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil int index = 0; // There can be multiple custom dictionary filters because it is a list. - for(final CustomDictionary customDictionary : filterProfile.getIdentifiers().getCustomDictionaries()) { + for(final CustomDictionary customDictionary : policy.getIdentifiers().getCustomDictionaries()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) if(customDictionary.isEnabled()) { @@ -1118,7 +1118,7 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil // All of the custom terms. final Set terms = new LinkedHashSet<>(); - // First, read the terms from the filter profile. + // First, read the terms from the policy. if(CollectionUtils.isNotEmpty(customDictionary.getTerms())) { terms.addAll(customDictionary.getTerms()); } @@ -1138,10 +1138,10 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil .withStrategies(customDictionary.getCustomDictionaryFilterStrategies()) .withAnonymizationService(new ZipCodeAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getZipCode().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getZipCode().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getZipCode().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getZipCode().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getZipCode().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getZipCode().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); @@ -1168,7 +1168,7 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil .withIgnored(customDictionary.getIgnored()) .withIgnoredFiles(customDictionary.getIgnoredFiles()) .withIgnoredPatterns(customDictionary.getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); @@ -1190,168 +1190,168 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil } else { - LOGGER.debug("Filter profile {} has no custom dictionaries.", filterProfile.getName()); + LOGGER.debug("Filter profile {} has no custom dictionaries.", policy.getName()); } // Lucene dictionary filters. - if(filterProfile.getIdentifiers().hasFilter(FilterType.LOCATION_CITY) && filterProfile.getIdentifiers().getCity().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.LOCATION_CITY) && policy.getIdentifiers().getCity().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getCity().getCityFilterStrategies()) + .withStrategies(policy.getIdentifiers().getCity().getCityFilterStrategies()) .withAnonymizationService(new CityAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getCity().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getCity().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getCity().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getCity().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getCity().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getCity().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getCity().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getCity().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getCity().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getCity().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, indexDirectory + "cities", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.LOCATION_COUNTY) && filterProfile.getIdentifiers().getCounty().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.LOCATION_COUNTY) && policy.getIdentifiers().getCounty().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getCounty().getCountyFilterStrategies()) + .withStrategies(policy.getIdentifiers().getCounty().getCountyFilterStrategies()) .withAnonymizationService(new CountyAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getCounty().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getCounty().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getCounty().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getCounty().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getCounty().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getCounty().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getCounty().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getCounty().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getCounty().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getCounty().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.LOCATION_COUNTY, filterConfiguration, indexDirectory + "counties", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.LOCATION_STATE) && filterProfile.getIdentifiers().getState().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.LOCATION_STATE) && policy.getIdentifiers().getState().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getState().getStateFilterStrategies()) + .withStrategies(policy.getIdentifiers().getState().getStateFilterStrategies()) .withAnonymizationService(new StateAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getState().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getState().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getState().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getState().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getState().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getState().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getState().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getState().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getState().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getState().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.LOCATION_STATE, filterConfiguration, indexDirectory + "states", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.HOSPITAL) && filterProfile.getIdentifiers().getHospital().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.HOSPITAL) && policy.getIdentifiers().getHospital().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getHospital().getHospitalFilterStrategies()) + .withStrategies(policy.getIdentifiers().getHospital().getHospitalFilterStrategies()) .withAnonymizationService(new HospitalAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getHospital().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getHospital().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getHospital().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getHospital().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getHospital().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getHospital().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getHospital().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getHospital().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getHospital().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getHospital().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.HOSPITAL, filterConfiguration, indexDirectory + "hospitals", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.HOSPITAL_ABBREVIATION) && filterProfile.getIdentifiers().getHospitalAbbreviation().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.HOSPITAL_ABBREVIATION) && policy.getIdentifiers().getHospitalAbbreviation().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getHospitalAbbreviation().getHospitalAbbreviationFilterStrategies()) + .withStrategies(policy.getIdentifiers().getHospitalAbbreviation().getHospitalAbbreviationFilterStrategies()) .withAnonymizationService(new HospitalAbbreviationAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getHospitalAbbreviation().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getHospitalAbbreviation().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getHospitalAbbreviation().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getHospitalAbbreviation().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getHospitalAbbreviation().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getHospitalAbbreviation().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getHospitalAbbreviation().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getHospitalAbbreviation().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getHospitalAbbreviation().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getHospitalAbbreviation().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.HOSPITAL_ABBREVIATION, filterConfiguration, indexDirectory + "hospital-abbreviations", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.FIRST_NAME) && filterProfile.getIdentifiers().getFirstName().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.FIRST_NAME) && policy.getIdentifiers().getFirstName().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getFirstName().getFirstNameFilterStrategies()) + .withStrategies(policy.getIdentifiers().getFirstName().getFirstNameFilterStrategies()) .withAnonymizationService(new PersonsAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getFirstName().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getFirstName().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getFirstName().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getFirstName().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getFirstName().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getFirstName().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getFirstName().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getFirstName().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getFirstName().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getFirstName().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, indexDirectory + "names", sensitivityLevel, capitalized)); } - if(filterProfile.getIdentifiers().hasFilter(FilterType.SURNAME) && filterProfile.getIdentifiers().getSurname().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.SURNAME) && policy.getIdentifiers().getSurname().isEnabled()) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getSurname().getSurnameFilterStrategies()) + .withStrategies(policy.getIdentifiers().getSurname().getSurnameFilterStrategies()) .withAnonymizationService(new SurnameAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getSurname().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getSurname().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getSurname().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getSurname().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getSurname().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getSurname().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); - final SensitivityLevel sensitivityLevel = filterProfile.getIdentifiers().getSurname().getSensitivityLevel(); - final boolean capitalized = filterProfile.getIdentifiers().getSurname().isCapitalized(); + final SensitivityLevel sensitivityLevel = policy.getIdentifiers().getSurname().getSensitivityLevel(); + final boolean capitalized = policy.getIdentifiers().getSurname().isCapitalized(); enabledFilters.add(new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, indexDirectory + "surnames", sensitivityLevel, capitalized)); @@ -1359,13 +1359,13 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil // Enable ID filter last since it is a pretty generic pattern that might also match SSN, et. al. - if(filterProfile.getIdentifiers().hasFilter(FilterType.IDENTIFIER)) { + if(policy.getIdentifiers().hasFilter(FilterType.IDENTIFIER)) { - final List identifiers = filterProfile.getIdentifiers().getIdentifiers(); + final List identifiers = policy.getIdentifiers().getIdentifiers(); for(final Identifier identifier : identifiers) { - // TODO: Add caching of the filter profile (see Age for example) + // TODO: Add caching of the policy (see Age for example) if(identifier.isEnabled()) { @@ -1376,7 +1376,7 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil .withIgnored(identifier.getIgnored()) .withIgnoredFiles(identifier.getIgnoredFiles()) .withIgnoredPatterns(identifier.getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); @@ -1394,20 +1394,20 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PERSON) && filterProfile.getIdentifiers().getPerson().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PERSON) && policy.getIdentifiers().getPerson().isEnabled()) { if(cache.containsKey(FilterType.PERSON)) { enabledFilters.add(cache.get(FilterType.PERSON)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPerson().getNerStrategies()) + .withStrategies(policy.getIdentifiers().getPerson().getNerStrategies()) .withAnonymizationService(new PersonsAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPerson().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPerson().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPerson().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPerson().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPerson().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPerson().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); @@ -1418,77 +1418,77 @@ private List getFiltersForFilterProfile(final FilterProfile filterProfil "PER", stats, metricsService, - filterProfile.getIdentifiers().getPerson().isRemovePunctuation(), - filterProfile.getIdentifiers().getPerson().getThresholds() + policy.getIdentifiers().getPerson().isRemovePunctuation(), + policy.getIdentifiers().getPerson().getThresholds() ); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PERSON, filter); + filterCache.get(policy.getName()).put(FilterType.PERSON, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PERSON_V2) && filterProfile.getIdentifiers().getPersonV2().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PERSON_V2) && policy.getIdentifiers().getPersonV2().isEnabled()) { if(cache.containsKey(FilterType.PERSON_V2)) { enabledFilters.add(cache.get(FilterType.PERSON_V2)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPersonV2().getNerStrategies()) + .withStrategies(policy.getIdentifiers().getPersonV2().getNerStrategies()) .withAnonymizationService(new PersonsAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPersonV2().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPersonV2().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPersonV2().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPersonV2().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPersonV2().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPersonV2().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PersonsV2Filter( filterConfiguration, - filterProfile.getIdentifiers().getPersonV2().getModel(), - filterProfile.getIdentifiers().getPersonV2().getVocab(), + policy.getIdentifiers().getPersonV2().getModel(), + policy.getIdentifiers().getPersonV2().getVocab(), stats, metricsService, - filterProfile.getIdentifiers().getPersonV2().getThresholds()); + policy.getIdentifiers().getPersonV2().getThresholds()); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PERSON_V2, filter); + filterCache.get(policy.getName()).put(FilterType.PERSON_V2, filter); } } - if(filterProfile.getIdentifiers().hasFilter(FilterType.PERSON_V3) && filterProfile.getIdentifiers().getPersonV3().isEnabled()) { + if(policy.getIdentifiers().hasFilter(FilterType.PERSON_V3) && policy.getIdentifiers().getPersonV3().isEnabled()) { if(cache.containsKey(FilterType.PERSON_V3)) { enabledFilters.add(cache.get(FilterType.PERSON_V3)); } else { final FilterConfiguration filterConfiguration = new FilterConfiguration.FilterConfigurationBuilder() - .withStrategies(filterProfile.getIdentifiers().getPersonV3().getNerStrategies()) + .withStrategies(policy.getIdentifiers().getPersonV3().getNerStrategies()) .withAnonymizationService(new PersonsAnonymizationService(anonymizationCacheService)) .withAlertService(alertService) - .withIgnored(filterProfile.getIdentifiers().getPersonV3().getIgnored()) - .withIgnoredFiles(filterProfile.getIdentifiers().getPersonV3().getIgnoredFiles()) - .withIgnoredPatterns(filterProfile.getIdentifiers().getPersonV3().getIgnoredPatterns()) - .withCrypto(filterProfile.getCrypto()) + .withIgnored(policy.getIdentifiers().getPersonV3().getIgnored()) + .withIgnoredFiles(policy.getIdentifiers().getPersonV3().getIgnoredFiles()) + .withIgnoredPatterns(policy.getIdentifiers().getPersonV3().getIgnoredPatterns()) + .withCrypto(policy.getCrypto()) .withWindowSize(windowSize) .withDocumentAnalysis(documentAnalysis) .build(); final Filter filter = new PersonsV3Filter( filterConfiguration, - filterProfile.getIdentifiers().getPersonV3().getModel(), + policy.getIdentifiers().getPersonV3().getModel(), stats, metricsService, - filterProfile.getIdentifiers().getPersonV3().getThresholds()); + policy.getIdentifiers().getPersonV3().getThresholds()); enabledFilters.add(filter); - filterCache.get(filterProfile.getName()).put(FilterType.PERSON_V3, filter); + filterCache.get(policy.getName()).put(FilterType.PERSON_V3, filter); } diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV2Filter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV2Filter.java index b8290467..f733ea11 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV2Filter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV2Filter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.services.MetricsService; import ai.philterd.phileas.service.ai.onnx.OnnxNer; import opennlp.tools.sentdetect.SentenceDetector; @@ -75,7 +75,7 @@ public PersonsV2Filter(FilterConfiguration filterConfiguration, } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { // Remove line breaks. input = input.replaceAll("\n", " "); @@ -91,7 +91,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d for(final Entity entity : entities) { final String[] window = getWindow(input, entity.getCharacterStart(), entity.getCharacterEnd()); - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, entity.getText(), window, entity.getConfidence(), classification, null); + final Replacement replacement = getReplacement(policy.getName(), context, documentId, entity.getText(), window, entity.getConfidence(), classification, null); final boolean isIgnored = ignored.contains(entity.getText()); final Span span = Span.make( @@ -120,7 +120,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d } @Override - public int getOccurrences(FilterProfile filterProfile, String input) throws Exception { + public int getOccurrences(Policy policy, String input) throws Exception { return 0; } diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV3Filter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV3Filter.java index 90a04352..39f4c880 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV3Filter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/opennlp/PersonsV3Filter.java @@ -21,7 +21,7 @@ import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.services.MetricsService; import opennlp.tools.namefind.NameFinderME; import opennlp.tools.namefind.TokenNameFinderModel; @@ -56,7 +56,7 @@ public PersonsV3Filter(FilterConfiguration filterConfiguration, } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { // The final list of spans identified in the text. final List spans = new LinkedList<>(); @@ -79,7 +79,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d final int end = start + text.length(); final String[] window = getWindow(input, start, end); - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, text, window, openNlpSpan.getProb(), classification, null); + final Replacement replacement = getReplacement(policy.getName(), context, documentId, text, window, openNlpSpan.getProb(), classification, null); final boolean isIgnored = ignored.contains(text); final Span span = Span.make( @@ -108,7 +108,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d } @Override - public int getOccurrences(FilterProfile filterProfile, String input) throws Exception { + public int getOccurrences(Policy policy, String input) throws Exception { return 0; } diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/python/PersonsV1Filter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/python/PersonsV1Filter.java index 38f6eef2..fc9c452c 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/python/PersonsV1Filter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/ai/python/PersonsV1Filter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.services.MetricsService; import okhttp3.ConnectionPool; import okhttp3.OkHttpClient; @@ -93,7 +93,7 @@ public PersonsV1Filter(FilterConfiguration filterConfiguration, } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { final List spans = new LinkedList<>(); @@ -124,7 +124,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d // Get the window of text surrounding the token. final String[] window = getWindow(input, phileasSpan.getStart(), phileasSpan.getEnd()); - final Span span = createSpan(filterProfile.getName(), input, context, documentId, phileasSpan.getText(), + final Span span = createSpan(policy.getName(), input, context, documentId, phileasSpan.getText(), window, phileasSpan.getTag(), phileasSpan.getStart(), phileasSpan.getEnd(), phileasSpan.getScore()); // Span will be null if no span was created due to it being excluded. @@ -159,16 +159,16 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d } @Override - public int getOccurrences(FilterProfile filterProfile, String input) throws Exception { + public int getOccurrences(Policy policy, String input) throws Exception { - return filter(filterProfile, "none", "none", 0, input).getSpans().size(); + return filter(policy, "none", "none", 0, input).getSpans().size(); } - private Span createSpan(String filterProfile, String input, String context, String documentId, String text, + private Span createSpan(String policy, String input, String context, String documentId, String text, String[] window, String classification, int start, int end, double confidence) throws Exception { - final Replacement replacement = getReplacement(filterProfile, context, documentId, text, window, confidence, classification, null); + final Replacement replacement = getReplacement(policy, context, documentId, text, window, confidence, classification, null); if(StringUtils.equals(replacement.getReplacement(), text)) { diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/custom/PhoneNumberRulesFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/custom/PhoneNumberRulesFilter.java index faa86fd4..dd75c188 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/custom/PhoneNumberRulesFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/custom/PhoneNumberRulesFilter.java @@ -23,7 +23,7 @@ import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.LinkedList; @@ -53,11 +53,11 @@ public PhoneNumberRulesFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { final List spans = new LinkedList<>(); - if(filterProfile.getIdentifiers().hasFilter(filterType)) { + if(policy.getIdentifiers().hasFilter(filterType)) { final Iterable matches = phoneUtil.findNumbers(input, "US", PhoneNumberUtil.Leniency.POSSIBLE, 1); @@ -79,7 +79,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d final String[] window = getWindow(input, match.start(), match.end()); final String classification = ""; - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, text, window, confidence, classification, null); + final Replacement replacement = getReplacement(policy.getName(), context, documentId, text, window, confidence, classification, null); final boolean isIgnored = ignored.contains(text); spans.add(Span.make(match.start(), match.end(), getFilterType(), context, documentId, confidence, text, replacement.getReplacement(), replacement.getSalt(), isIgnored, window)); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/AgeFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/AgeFilter.java index 48a7ace8..d5a690d5 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/AgeFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/AgeFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.Arrays; import java.util.HashSet; @@ -58,9 +58,9 @@ public AgeFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); final List nonOverlappingSpans = Span.dropOverlappingSpans(spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BankRoutingNumberFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BankRoutingNumberFilter.java index 601dfd4f..b583517b 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BankRoutingNumberFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BankRoutingNumberFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.LinkedList; @@ -46,9 +46,9 @@ public BankRoutingNumberFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); final List nonOverlappingSpans = Span.dropOverlappingSpans(spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BitcoinAddressFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BitcoinAddressFilter.java index a9ac1dee..2e7fdac6 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BitcoinAddressFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/BitcoinAddressFilter.java @@ -22,16 +22,10 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.profile.IgnoredPattern; -import ai.philterd.phileas.model.profile.filters.strategies.AbstractFilterStrategy; -import ai.philterd.phileas.model.services.AlertService; -import ai.philterd.phileas.model.services.AnonymizationService; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.regex.Pattern; public class BitcoinAddressFilter extends RegexFilter { @@ -53,9 +47,9 @@ public BitcoinAddressFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CreditCardFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CreditCardFilter.java index c0a2d963..386258bc 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CreditCardFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CreditCardFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import org.apache.commons.validator.routines.checkdigit.LuhnCheckDigit; import java.util.HashSet; @@ -59,9 +59,9 @@ public CreditCardFilter(FilterConfiguration filterConfiguration, boolean onlyVal } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); final List validSpans = new LinkedList<>(); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CurrencyFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CurrencyFilter.java index c09bd31d..4a7c0891 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CurrencyFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/CurrencyFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -46,9 +46,9 @@ public CurrencyFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); final List nonOverlappingSpans = Span.dropOverlappingSpans(spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DateFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DateFilter.java index 6c602e45..ee4ae945 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DateFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DateFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.services.SpanValidator; import java.util.Arrays; @@ -58,11 +58,11 @@ public DateFilter(FilterConfiguration filterConfiguration, boolean onlyValidDate } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { final List spans = new LinkedList<>(); - final List rawSpans = findSpans(filterProfile, analyzer, input, context, documentId); + final List rawSpans = findSpans(policy, analyzer, input, context, documentId); if(onlyValidDates) { diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DriversLicenseFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DriversLicenseFilter.java index a7e5e991..183f1fdc 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DriversLicenseFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/DriversLicenseFilter.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.filter.FilterConfiguration; import ai.philterd.phileas.model.filter.rules.regex.RegexFilter; import ai.philterd.phileas.model.objects.*; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashMap; import java.util.HashSet; @@ -105,9 +105,9 @@ public DriversLicenseFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, Span.dropOverlappingSpans(spans)); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/EmailAddressFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/EmailAddressFilter.java index a3c377cd..402c5f14 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/EmailAddressFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/EmailAddressFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -45,9 +45,9 @@ public EmailAddressFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IbanCodeFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IbanCodeFilter.java index 5a5a7f7e..866c2132 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IbanCodeFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IbanCodeFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import org.apache.commons.validator.routines.IBANValidator; import java.util.HashSet; @@ -65,9 +65,9 @@ public IbanCodeFilter(FilterConfiguration filterConfiguration, boolean validate, } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); final List validSpans = new LinkedList<>(); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IdentifierFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IdentifierFilter.java index 6b169eed..6661e1f2 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IdentifierFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IdentifierFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -41,8 +41,8 @@ public IdentifierFilter(FilterConfiguration filterConfiguration, String classifi pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); } - // TODO: Expose initialConfidence via the filter profile. - // TODO: Expose the contextual terms via the filter profile. + // TODO: Expose initialConfidence via the policy. + // TODO: Expose the contextual terms via the policy. final FilterPattern id1 = new FilterPattern.FilterPatternBuilder(pattern, 0.90) .withClassification(classification) .withGroupNumber(groupNumber) @@ -56,9 +56,9 @@ public IdentifierFilter(FilterConfiguration filterConfiguration, String classifi } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IpAddressFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IpAddressFilter.java index 753a7dc6..b438f8a6 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IpAddressFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/IpAddressFilter.java @@ -22,16 +22,10 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.profile.IgnoredPattern; -import ai.philterd.phileas.model.profile.filters.strategies.AbstractFilterStrategy; -import ai.philterd.phileas.model.services.AlertService; -import ai.philterd.phileas.model.services.AnonymizationService; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.regex.Pattern; public class IpAddressFilter extends RegexFilter { @@ -108,9 +102,9 @@ public IpAddressFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/MacAddressFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/MacAddressFilter.java index 4876983a..77142e0b 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/MacAddressFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/MacAddressFilter.java @@ -22,16 +22,10 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.profile.IgnoredPattern; -import ai.philterd.phileas.model.profile.filters.strategies.AbstractFilterStrategy; -import ai.philterd.phileas.model.services.AlertService; -import ai.philterd.phileas.model.services.AnonymizationService; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.regex.Pattern; public class MacAddressFilter extends RegexFilter { @@ -51,9 +45,9 @@ public MacAddressFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PassportNumberFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PassportNumberFilter.java index 32bd54a5..c1c51407 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PassportNumberFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PassportNumberFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -101,9 +101,9 @@ public PassportNumberFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhoneNumberExtensionFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhoneNumberExtensionFilter.java index 22ab5615..824a8f3c 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhoneNumberExtensionFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhoneNumberExtensionFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -46,9 +46,9 @@ public PhoneNumberExtensionFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhysicianNameFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhysicianNameFilter.java index 19fd5b21..04f69332 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhysicianNameFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/PhysicianNameFilter.java @@ -22,17 +22,10 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.profile.IgnoredPattern; -import ai.philterd.phileas.model.profile.filters.strategies.AbstractFilterStrategy; -import ai.philterd.phileas.model.services.AlertService; -import ai.philterd.phileas.model.services.AnonymizationService; +import ai.philterd.phileas.model.profile.Policy; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.shingle.ShingleFilter; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import java.io.IOException; @@ -40,7 +33,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -73,7 +65,7 @@ public PhysicianNameFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { // \b([A-Z][A-Za-z'\s+]+)(,|\s)?([A-Z][A-Za-z'\s+]+(,|\s))?([A-Z][A-Za-z'\s+]+(,|\s)?(MD|PhD))\b @@ -101,7 +93,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d final FilterPattern filterPattern = new FilterPattern.FilterPatternBuilder(candidatePattern, 0.90).build(); this.analyzer = new Analyzer(contextualTerms, filterPattern); - final List patternSpans = findSpans(filterProfile, analyzer, input, context, documentId); + final List patternSpans = findSpans(policy, analyzer, input, context, documentId); spans.addAll(patternSpans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SectionFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SectionFilter.java index 1f809153..9d7c8170 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SectionFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SectionFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -44,9 +44,9 @@ public SectionFilter(FilterConfiguration filterConfiguration, String startPatter } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SsnFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SsnFilter.java index 120811f1..4a63144c 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SsnFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/SsnFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -50,9 +50,9 @@ public SsnFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StateAbbreviationFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StateAbbreviationFilter.java index af768e66..2731b263 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StateAbbreviationFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StateAbbreviationFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.LinkedList; @@ -106,9 +106,9 @@ public StateAbbreviationFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StreetAddressFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StreetAddressFilter.java index b02f6a5a..827c2d29 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StreetAddressFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/StreetAddressFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -45,9 +45,9 @@ public StreetAddressFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/TrackingNumberFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/TrackingNumberFilter.java index 00517345..d574da56 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/TrackingNumberFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/TrackingNumberFilter.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.filter.FilterConfiguration; import ai.philterd.phileas.model.filter.rules.regex.RegexFilter; import ai.philterd.phileas.model.objects.*; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.*; import java.util.regex.Pattern; @@ -135,7 +135,7 @@ public TrackingNumberFilter(FilterConfiguration filterConfiguration, boolean ups } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { final List classifications = new LinkedList<>(); @@ -154,7 +154,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d final Map> restrictions = new HashMap<>(); restrictions.put(Restriction.CLASSIFICATION, classifications); - final List spans = findSpans(filterProfile, analyzer, input, context, documentId, restrictions); + final List spans = findSpans(policy, analyzer, input, context, documentId, restrictions); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/UrlFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/UrlFilter.java index a75e9c42..4f3b4ccc 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/UrlFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/UrlFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -65,9 +65,9 @@ public UrlFilter(FilterConfiguration filterConfiguration, boolean requireHttpWww } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/VinFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/VinFilter.java index 76d8b6ac..6b9e0d92 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/VinFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/VinFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import org.apache.commons.collections.CollectionUtils; import java.util.HashSet; @@ -49,9 +49,9 @@ public VinFilter(FilterConfiguration filterConfiguration) { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); CollectionUtils.filter(spans, object -> { Span s = (Span) object; diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/ZipCodeFilter.java b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/ZipCodeFilter.java index 6252ed84..b7db13c1 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/ZipCodeFilter.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/filters/regex/ZipCodeFilter.java @@ -22,7 +22,7 @@ import ai.philterd.phileas.model.objects.FilterPattern; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.HashSet; import java.util.List; @@ -59,9 +59,9 @@ public ZipCodeFilter(FilterConfiguration filterConfiguration, boolean requireDel } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception { - final List spans = findSpans(filterProfile, analyzer, input, context, documentId); + final List spans = findSpans(policy, analyzer, input, context, documentId); return new FilterResult(context, documentId, spans); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java index be859667..b8416162 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTests.java @@ -20,7 +20,7 @@ import ai.philterd.phileas.configuration.PhileasConfiguration; import ai.philterd.phileas.model.enums.MimeType; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.filters.CustomDictionary; import ai.philterd.phileas.model.profile.filters.strategies.custom.CustomDictionaryFilterStrategy; import ai.philterd.phileas.model.responses.BinaryDocumentFilterResponse; @@ -115,7 +115,7 @@ public void endToEnd1() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -140,7 +140,7 @@ public void endToEnd2() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -165,7 +165,7 @@ public void endToEnd3() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -190,7 +190,7 @@ public void endToEnd4() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -215,7 +215,7 @@ public void endToEnd5() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -240,7 +240,7 @@ public void endToEnd6() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -265,7 +265,7 @@ public void endToEnd7() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -290,7 +290,7 @@ public void endToEnd8() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -316,7 +316,7 @@ public void endToEnd9() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -341,7 +341,7 @@ public void endToEnd10() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -371,7 +371,7 @@ public void endToEnd11() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -398,7 +398,7 @@ public void endToEnd12() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -430,7 +430,7 @@ public void endToEnd13() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -460,7 +460,7 @@ public void endToEnd14() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -490,7 +490,7 @@ public void endToEnd15() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -520,7 +520,7 @@ public void endToEnd16() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - final String profile = gson.toJson(getFilterProfileJustIdentifier("default")); + final String profile = gson.toJson(getPolicyJustIdentifier("default")); LOGGER.info(profile); FileUtils.writeStringToFile(file, profile); @@ -552,7 +552,7 @@ public void endToEnd17() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "streetaddress.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - final String profile = gson.toJson(getFilterProfileJustStreetAddress("streetaddress")); + final String profile = gson.toJson(getPolicyJustStreetAddress("streetaddress")); LOGGER.info(profile); FileUtils.writeStringToFile(file, profile); @@ -586,13 +586,13 @@ public void endToEndUsingCustomDictionary() throws Exception { customDictionary.setTerms(Arrays.asList("george", "samuel")); customDictionary.setFuzzy(false); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName("custom-dictionary"); - filterProfile.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); + final Policy policy = new Policy(); + policy.setName("custom-dictionary"); + policy.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); - FileUtils.writeStringToFile(file, gson.toJson(filterProfile), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(policy), Charset.defaultCharset()); LOGGER.info("Filter profile written to {}", file.getAbsolutePath()); final Properties properties = new Properties(); @@ -630,12 +630,12 @@ public void endToEndUsingCustomDictionaryFileLuceneFilter() throws Exception { customDictionary.setFuzzy(false); customDictionary.setSensitivity("low"); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName("custom-dictionary"); - filterProfile.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); + final Policy policy = new Policy(); + policy.setName("custom-dictionary"); + policy.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); - FileUtils.writeStringToFile(file, gson.toJson(filterProfile), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(policy), Charset.defaultCharset()); LOGGER.info("Filter profile written to {}", file.getAbsolutePath()); final Properties properties = new Properties(); @@ -671,12 +671,12 @@ public void endToEndUsingCustomDictionaryFileBloomFilter() throws Exception { customDictionary.setClassification("names"); customDictionary.setFuzzy(false); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName("custom-dictionary-bloom"); - filterProfile.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); + final Policy policy = new Policy(); + policy.setName("custom-dictionary-bloom"); + policy.getIdentifiers().setCustomDictionaries(Arrays.asList(customDictionary)); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); - FileUtils.writeStringToFile(file, gson.toJson(filterProfile), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(policy), Charset.defaultCharset()); LOGGER.info("Filter profile written to {}", file.getAbsolutePath()); final Properties properties = new Properties(); @@ -702,7 +702,7 @@ public void endToEndWithoutDocumentId() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -723,17 +723,17 @@ public void endToEndWithoutDocumentId() throws Exception { } @Test - public void endToEndMultipleFilterProfiles() throws Exception { + public void endToEndMultiplePolicys() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file1 = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file1.getAbsolutePath()); - FileUtils.writeStringToFile(file1, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file1, gson.toJson(getPolicy("default")), Charset.defaultCharset()); final File file2 = Paths.get(temp.toFile().getAbsolutePath(), "justcreditcard.json").toFile(); LOGGER.info("Writing profile to {}", file2.getAbsolutePath()); - FileUtils.writeStringToFile(file2, gson.toJson(getFilterProfileJustCreditCard("justcreditcard")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file2, gson.toJson(getPolicyJustCreditCard("justcreditcard")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -758,7 +758,7 @@ public void endToEndJustCreditCard() throws Exception { final File file2 = Paths.get(temp.toFile().getAbsolutePath(), "justcreditcard.json").toFile(); LOGGER.info("Writing profile to {}", file2.getAbsolutePath()); - FileUtils.writeStringToFile(file2, gson.toJson(getFilterProfileJustCreditCard("justcreditcard")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file2, gson.toJson(getPolicyJustCreditCard("justcreditcard")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -783,7 +783,7 @@ public void endToEndJustCreditCardWithIgnoredTerms() throws Exception { final File file2 = Paths.get(temp.toFile().getAbsolutePath(), "justcreditcard.json").toFile(); LOGGER.info("Writing profile to {}", file2.getAbsolutePath()); - FileUtils.writeStringToFile(file2, gson.toJson(getFilterProfileJustCreditCard("justcreditcard")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file2, gson.toJson(getPolicyJustCreditCard("justcreditcard")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -807,7 +807,7 @@ public void endToEndWithFilterSpecificIgnoredTerms() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfileZipCodeWithIgnored("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicyZipCodeWithIgnored("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -831,7 +831,7 @@ public void endToEndWithFilterIgnoredTermsFromFile() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file.getAbsolutePath()); - FileUtils.writeStringToFile(file, gson.toJson(getFilterProfileZipCodeWithIgnoredFromFile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file, gson.toJson(getPolicyZipCodeWithIgnoredFromFile("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -850,13 +850,13 @@ public void endToEndWithFilterIgnoredTermsFromFile() throws Exception { } @Test - public void endToEndNonexistentFilterProfile() throws Exception { + public void endToEndNonexistentPolicy() throws Exception { final Path temp = Files.createTempDirectory("philter"); final File file1 = Paths.get(temp.toFile().getAbsolutePath(), "default.json").toFile(); LOGGER.info("Writing profile to {}", file1.getAbsolutePath()); - FileUtils.writeStringToFile(file1, gson.toJson(getFilterProfile("default")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file1, gson.toJson(getPolicy("default")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java index c7e2c6e6..78ff1c5e 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/EndToEndTestsHelper.java @@ -15,7 +15,7 @@ */ package ai.philterd.test.phileas.services; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Identifiers; import ai.philterd.phileas.model.profile.Ignored; import ai.philterd.phileas.model.profile.filters.*; @@ -31,7 +31,6 @@ import ai.philterd.phileas.model.profile.filters.strategies.dynamic.SurnameFilterStrategy; import ai.philterd.phileas.model.profile.filters.strategies.rules.*; import org.apache.commons.io.FileUtils; -import org.jsoup.select.Evaluator; import java.io.File; import java.io.IOException; @@ -44,7 +43,7 @@ public class EndToEndTestsHelper { - public static FilterProfile getFilterProfileZipCodeWithIgnored(String filterProfileName) throws IOException { + public static Policy getPolicyZipCodeWithIgnored(String policyName) throws IOException { Set ignored = new HashSet<>(); ignored.add("90210"); @@ -65,15 +64,15 @@ public static FilterProfile getFilterProfileZipCodeWithIgnored(String filterProf identifiers.setSsn(ssn); identifiers.setZipCode(zipCode); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getFilterProfileZipCodeWithIgnoredFromFile(String filterProfileName) throws IOException { + public static Policy getPolicyZipCodeWithIgnoredFromFile(String policyName) throws IOException { // Copy file to temp directory. final File file = File.createTempFile("philter", "ignore"); @@ -98,15 +97,15 @@ public static FilterProfile getFilterProfileZipCodeWithIgnoredFromFile(String fi identifiers.setSsn(ssn); identifiers.setZipCode(zipCode); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getPdfFilterProfile(String filterProfileName) throws IOException { + public static Policy getPdfPolicy(String policyName) throws IOException { ZipCodeFilterStrategy zipCodeFilterStrategy = new ZipCodeFilterStrategy(); zipCodeFilterStrategy.setTruncateDigits(2); @@ -126,15 +125,15 @@ public static FilterProfile getPdfFilterProfile(String filterProfileName) throws identifiers.setCustomDictionaries(Arrays.asList(customDictionary)); identifiers.setZipCode(zipCode); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getPdfFilterWithPersonProfile(String filterProfileName) throws URISyntaxException { + public static Policy getPdfFilterWithPersonProfile(String policyName) throws URISyntaxException { final File model = new File(EndToEndTestsHelper.class.getClassLoader().getResource("ner/model.onnx").toURI()); final File vocab = new File(EndToEndTestsHelper.class.getClassLoader().getResource("ner/vocab.txt").toURI()); @@ -146,15 +145,15 @@ public static FilterProfile getPdfFilterWithPersonProfile(String filterProfileNa Identifiers identifiers = new Identifiers(); identifiers.setPersonV2(personV2); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getFilterProfileJustCreditCard(String filterProfileName) throws IOException { + public static Policy getPolicyJustCreditCard(String policyName) throws IOException { CreditCardFilterStrategy creditCardFilterStrategy = new CreditCardFilterStrategy(); @@ -167,16 +166,16 @@ public static FilterProfile getFilterProfileJustCreditCard(String filterProfileN Ignored ignored = new Ignored(); ignored.setTerms(Arrays.asList("4121742025464400", "12341341234", "2423543545")); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); - filterProfile.setIgnored(Arrays.asList(ignored)); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); + policy.setIgnored(Arrays.asList(ignored)); - return filterProfile; + return policy; } - public static FilterProfile getFilterProfile(String filterProfileName) throws IOException, URISyntaxException { + public static Policy getPolicy(String policyName) throws IOException, URISyntaxException { AgeFilterStrategy ageFilterStrategy = new AgeFilterStrategy(); @@ -324,15 +323,15 @@ public static FilterProfile getFilterProfile(String filterProfileName) throws IO identifiers.setState(state); identifiers.setSurname(surname);*/ - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getFilterProfileJustIdentifier(String filterProfileName) { + public static Policy getPolicyJustIdentifier(String policyName) { Identifier identifier1 = new Identifier(); identifier1.setIdentifierFilterStrategies(Arrays.asList(new IdentifierFilterStrategy())); @@ -343,15 +342,15 @@ public static FilterProfile getFilterProfileJustIdentifier(String filterProfileN identifiers.setIdentifiers(Arrays.asList(identifier1)); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } - public static FilterProfile getFilterProfileJustStreetAddress(String filterProfileName) { + public static Policy getPolicyJustStreetAddress(String policyName) { StreetAddressFilterStrategy streetAddressFilterStrategy = new StreetAddressFilterStrategy(); @@ -361,11 +360,11 @@ public static FilterProfile getFilterProfileJustStreetAddress(String filterProfi Identifiers identifiers = new Identifiers(); identifiers.setStreetAddress(streetAddress); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(filterProfileName); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(policyName); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/PhileasFilterServiceTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/PhileasFilterServiceTest.java index ac2edffc..d05835a1 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/PhileasFilterServiceTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/PhileasFilterServiceTest.java @@ -20,7 +20,7 @@ import ai.philterd.phileas.configuration.PhileasConfiguration; import ai.philterd.phileas.model.enums.MimeType; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Ignored; import ai.philterd.phileas.model.responses.BinaryDocumentFilterResponse; import ai.philterd.phileas.model.serializers.PlaceholderDeserializer; @@ -47,8 +47,8 @@ import java.util.List; import java.util.Properties; -import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getFilterProfile; -import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPdfFilterProfile; +import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPolicy; +import static ai.philterd.test.phileas.services.EndToEndTestsHelper.getPdfPolicy; public class PhileasFilterServiceTest { @@ -67,33 +67,33 @@ public void before() { } @Test - public void filterProfile() throws IOException, URISyntaxException { + public void policy() throws IOException, URISyntaxException { - final FilterProfile filterProfile = getFilterProfile("default"); - final String json = gson.toJson(filterProfile); + final Policy policy = getPolicy("default"); + final String json = gson.toJson(policy); LOGGER.info(json); - final FilterProfile deserialized = gson.fromJson(json, FilterProfile.class); + final Policy deserialized = gson.fromJson(json, Policy.class); Assertions.assertEquals("default", deserialized.getName()); } @Test - public void filterProfileWithPlaceholder() throws IOException, URISyntaxException { + public void policyWithPlaceholder() throws IOException, URISyntaxException { final Ignored ignored = new Ignored(); ignored.setTerms(Arrays.asList("john", "jeff", "${USER}")); - final FilterProfile filterProfile = getFilterProfile("placeholder"); - filterProfile.setIgnored(Arrays.asList(ignored)); - final String json = gson.toJson(filterProfile); + final Policy policy = getPolicy("placeholder"); + policy.setIgnored(Arrays.asList(ignored)); + final String json = gson.toJson(policy); LOGGER.info(json); - final FilterProfile deserialized = gson.fromJson(json, FilterProfile.class); + final Policy deserialized = gson.fromJson(json, Policy.class); Assertions.assertEquals("placeholder", deserialized.getName()); - Assertions.assertEquals(3, filterProfile.getIgnored().get(0).getTerms().size()); + Assertions.assertEquals(3, policy.getIgnored().get(0).getTerms().size()); Assertions.assertTrue(CollectionUtils.isNotEmpty(deserialized.getIgnored().get(0).getTerms())); } @@ -111,7 +111,7 @@ public void pdf1() throws Exception { final File file1 = Paths.get(temp.toFile().getAbsolutePath(), "pdf.json").toFile(); LOGGER.info("Writing profile to {}", file1.getAbsolutePath()); - FileUtils.writeStringToFile(file1, gson.toJson(getPdfFilterProfile("pdf")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file1, gson.toJson(getPdfPolicy("pdf")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); @@ -148,7 +148,7 @@ public void pdf2() throws Exception { final File file1 = Paths.get(temp.toFile().getAbsolutePath(), "pdf.json").toFile(); LOGGER.info("Writing profile to {}", file1.getAbsolutePath()); - FileUtils.writeStringToFile(file1, gson.toJson(getPdfFilterProfile("pdf")), Charset.defaultCharset()); + FileUtils.writeStringToFile(file1, gson.toJson(getPdfPolicy("pdf")), Charset.defaultCharset()); Properties properties = new Properties(); properties.setProperty("indexes.directory", INDEXES_DIRECTORY); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AbstractFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AbstractFilterTest.java index e84177e5..72bcdc19 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AbstractFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AbstractFilterTest.java @@ -18,7 +18,7 @@ import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.enums.SensitivityLevel; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Identifiers; import ai.philterd.phileas.model.profile.filters.*; import ai.philterd.phileas.model.profile.filters.strategies.custom.CustomDictionaryFilterStrategy; @@ -40,10 +40,10 @@ public abstract class AbstractFilterTest { protected final int windowSize = 3; /** - * Gets a {@link FilterProfile} where all non-deterministic filters use the given {@link SensitivityLevel}. - * @return A {@link FilterProfile} where all non-deterministic filters use the given {@link SensitivityLevel}. + * Gets a {@link Policy} where all non-deterministic filters use the given {@link SensitivityLevel}. + * @return A {@link Policy} where all non-deterministic filters use the given {@link SensitivityLevel}. */ - public FilterProfile getFilterProfile() throws IOException { + public Policy getPolicy() throws IOException { CustomDictionary customDictionary = new CustomDictionary(); customDictionary.setCustomDictionaryFilterStrategies(Arrays.asList(new CustomDictionaryFilterStrategy())); @@ -189,11 +189,11 @@ public FilterProfile getFilterProfile() throws IOException { identifiers.setState(state); identifiers.setSurname(surname); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName("default"); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName("default"); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AgeFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AgeFilterTest.java index 25f460b2..af5160f1 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AgeFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/AgeFilterTest.java @@ -56,7 +56,7 @@ public void filterIgnoredPattern1() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 35years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 35years old."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(filterResult.getSpans().get(0).isIgnored()); @@ -81,7 +81,7 @@ public void filterIgnoredCaseSensitive() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 35yEaRs old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 35yEaRs old."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(filterResult.getSpans().get(0).isIgnored()); @@ -102,7 +102,7 @@ public void filter0() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 3.5years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 3.5years old."); showSpans(filterResult.getSpans()); @@ -126,7 +126,7 @@ public void filter1() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 3.5years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 3.5years old."); showSpans(filterResult.getSpans()); @@ -148,7 +148,7 @@ public void filter2() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient age is 3.yrs."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient age is 3.yrs."); showSpans(filterResult.getSpans()); @@ -170,7 +170,7 @@ public void filter3() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient age is 3yrs."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient age is 3yrs."); showSpans(filterResult.getSpans()); @@ -192,7 +192,7 @@ public void filter4() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 3.5yrs old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 3.5yrs old."); showSpans(filterResult.getSpans()); @@ -214,7 +214,7 @@ public void filter5() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the patient is 39yrs. old"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the patient is 39yrs. old"); showSpans(filterResult.getSpans()); @@ -236,7 +236,7 @@ public void filter6() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "she is aged 39"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "she is aged 39"); showSpans(filterResult.getSpans()); @@ -258,7 +258,7 @@ public void filter7() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "she is age 39"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "she is age 39"); showSpans(filterResult.getSpans()); @@ -280,7 +280,7 @@ public void filter8() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "she is age 39.5"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "she is age 39.5"); showSpans(filterResult.getSpans()); @@ -302,7 +302,7 @@ public void filter9() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Patient Timothy Hook is 72 Yr. old male lives alone."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Patient Timothy Hook is 72 Yr. old male lives alone."); showSpans(filterResult.getSpans()); @@ -326,7 +326,7 @@ public void filter10() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Cari Morris is 75 yo female alert and oriented x’s3 with some mild memory loss."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Cari Morris is 75 yo female alert and oriented x’s3 with some mild memory loss."); showSpans(filterResult.getSpans()); @@ -350,7 +350,7 @@ public void filter11() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Had symptoms for the past 10 years"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Had symptoms for the past 10 years"); showSpans(filterResult.getSpans()); @@ -372,7 +372,7 @@ public void filter12() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "She is a 22-year-old female"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "She is a 22-year-old female"); showSpans(filterResult.getSpans()); @@ -396,7 +396,7 @@ public void filter13() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Admit age: 69 years"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Admit age: 69 years"); showSpans(filterResult.getSpans()); @@ -420,7 +420,7 @@ public void filter14() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Female Admit Age: 69 years"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Female Admit Age: 69 years"); showSpans(filterResult.getSpans()); @@ -444,7 +444,7 @@ public void filter15() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); showSpans(filterResult.getSpans()); @@ -468,7 +468,7 @@ public void filter16() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "patient is 61 y/o and"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "patient is 61 y/o and"); showSpans(filterResult.getSpans()); @@ -492,7 +492,7 @@ public void filter17() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "patient is 161 y/o and"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "patient is 161 y/o and"); showSpans(filterResult.getSpans()); @@ -516,7 +516,7 @@ public void filter18() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "patient is 4161 y/o and"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "patient is 4161 y/o and"); showSpans(filterResult.getSpans()); @@ -540,7 +540,7 @@ public void filter_subpoena() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); showSpans(filterResult.getSpans()); @@ -568,7 +568,7 @@ public void filter_no_document_type() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Female Admit Age: 69 years\n"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BankRoutingNumberFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BankRoutingNumberFilterTest.java index 9fe3403b..cd8da479 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BankRoutingNumberFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BankRoutingNumberFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final BankRoutingNumberFilter filter = new BankRoutingNumberFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the routing number is 111000025 patient is 3.5years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the routing number is 111000025 patient is 3.5years old."); showSpans(filterResult.getSpans()); @@ -67,7 +67,7 @@ public void filter2() throws Exception { final BankRoutingNumberFilter filter = new BankRoutingNumberFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the routing number is 111007025 patient is 3.5years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the routing number is 111007025 patient is 3.5years old."); showSpans(filterResult.getSpans()); @@ -87,7 +87,7 @@ public void filter3() throws Exception { final BankRoutingNumberFilter filter = new BankRoutingNumberFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the routing number is 1131007025 patient is 3.5years old."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the routing number is 1131007025 patient is 3.5years old."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BitcoinAddressFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BitcoinAddressFilterTest.java index e63c54d6..d7cdab20 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BitcoinAddressFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BitcoinAddressFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final BitcoinAddressFilter filter = new BitcoinAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the address is 127NVqnjf8gB9BFAW2dnQeM6wqmy1gbGtv."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the address is 127NVqnjf8gB9BFAW2dnQeM6wqmy1gbGtv."); showSpans(filterResult.getSpans()); @@ -68,7 +68,7 @@ public void filter2() throws Exception { final BitcoinAddressFilter filter = new BitcoinAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the address is 12qnjf8FAW2dnQeM6wqmy1gbGtv."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the address is 12qnjf8FAW2dnQeM6wqmy1gbGtv."); showSpans(filterResult.getSpans()); @@ -91,7 +91,7 @@ public void filter3() throws Exception { final BitcoinAddressFilter filter = new BitcoinAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the address is 126wqmy1gbGtv."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the address is 126wqmy1gbGtv."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BloomFilterDictionaryFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BloomFilterDictionaryFilterTest.java index 69ed56f4..f93b337e 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BloomFilterDictionaryFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/BloomFilterDictionaryFilterTest.java @@ -52,7 +52,7 @@ public void filterDictionaryExactMatch() throws Exception { final Set names = new HashSet<>(Arrays.asList("george", "ted", "Bill", "john")); final BloomFilterDictionaryFilter filter = new BloomFilterDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, names, "none", 0.05); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "He lived with Bill in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "He lived with Bill in California."); showSpans(filterResult.getSpans()); @@ -75,7 +75,7 @@ public void filterDictionaryCaseInsensitiveMatch() throws Exception { final Set names = new HashSet<>(Arrays.asList("george", "ted", "bill", "john")); final BloomFilterDictionaryFilter filter = new BloomFilterDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, names, "none", 0.05); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "He lived with Bill in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "He lived with Bill in California."); showSpans(filterResult.getSpans()); @@ -98,7 +98,7 @@ public void filterDictionaryNoMatch() throws Exception { final Set names = new HashSet<>(Arrays.asList("george", "ted", "bill", "john")); final BloomFilterDictionaryFilter filter = new BloomFilterDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, names, "none", 0.05); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "He lived with Sam in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "He lived with Sam in California."); showSpans(filterResult.getSpans()); @@ -119,7 +119,7 @@ public void filterDictionaryPhraseMatch1() throws Exception { final Set names = new HashSet<>(Arrays.asList("george jones", "ted", "bill", "john")); final BloomFilterDictionaryFilter filter = new BloomFilterDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, names, "none", 0.05); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"He lived with george jones in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"He lived with george jones in California."); showSpans(filterResult.getSpans()); @@ -142,7 +142,7 @@ public void filterDictionaryPhraseMatch2() throws Exception { final Set names = new HashSet<>(Arrays.asList("george jones jr", "ted", "bill smith", "john")); final BloomFilterDictionaryFilter filter = new BloomFilterDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, names, "none", 0.05); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Bill Smith lived with george jones jr in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Bill Smith lived with george jones jr in California."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CityFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CityFilterTest.java index 0967450a..20cb5505 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CityFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CityFilterTest.java @@ -79,7 +79,7 @@ public void filterCitiesExactMatch() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Washington."); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Washington."); showSpans(filterResult.getSpans()); @@ -100,7 +100,7 @@ public void filterCitiesExactMatch2() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in New York."); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in New York."); showSpans(filterResult.getSpans()); @@ -123,7 +123,7 @@ public void filterCitiesLow() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Lived in Wshington"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Lived in Wshington"); showSpans(filterResult.getSpans()); @@ -143,7 +143,7 @@ public void filterCitiesMedium() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Wshington"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Wshington"); showSpans(filterResult.getSpans()); @@ -164,7 +164,7 @@ public void filterCitiesHigh() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_CITY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Lived in Wasinton"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Lived in Wasinton"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CountyFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CountyFilterTest.java index e44e2e8a..6e02b771 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CountyFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CountyFilterTest.java @@ -60,7 +60,7 @@ public void filterCountiesLow() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_COUNTY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Lived in Fyette"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Lived in Fyette"); showSpans(filterResult.getSpans()); @@ -80,7 +80,7 @@ public void filterCountiesMedium() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_COUNTY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Fyette"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Fyette"); showSpans(filterResult.getSpans()); @@ -103,7 +103,7 @@ public void filterCountiesHigh() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_COUNTY, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Fyette"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Fyette"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CreditCardFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CreditCardFilterTest.java index cc95e120..fd41fea2 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CreditCardFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CreditCardFilterTest.java @@ -47,51 +47,51 @@ public void filterCreditCardOnlyValid() throws Exception { // VISA - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4532613702852251 visa."); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4532613702852251 visa."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); Assertions.assertEquals("4532613702852251", filterResult.getSpans().get(0).getText()); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4556662764258031"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4556662764258031"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4929081870602661"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4929081870602661"); Assertions.assertEquals(1, filterResult.getSpans().size()); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4716-4366-8767-7438"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4716-4366-8767-7438"); Assertions.assertEquals(1, filterResult.getSpans().size()); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4556 5849 8186 7933"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4556 5849 8186 7933"); Assertions.assertEquals(1, filterResult.getSpans().size()); // AMEX - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 376454057275914"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 376454057275914"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 346009657106278."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 346009657106278."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.CREDIT_CARD)); // MASTERCARD - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 5567408136464012"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 5567408136464012"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 5100170632668801."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 5100170632668801."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); // DISCOVER - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 6011485579364263"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 6011485579364263"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 6011792597726344."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 6011792597726344."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); @@ -111,50 +111,50 @@ public void filterCreditCardValidAndInvalid() throws Exception { // VISA - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 1234567812345678 visa."); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 1234567812345678 visa."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 0000000000000000"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 0000000000000000"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 9876543219876543"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 9876543219876543"); Assertions.assertEquals(1, filterResult.getSpans().size()); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 9876-5432-1987-6543"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 9876-5432-1987-6543"); Assertions.assertEquals(1, filterResult.getSpans().size()); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 4556 6627 6425 8000"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 4556 6627 6425 8000"); Assertions.assertEquals(1, filterResult.getSpans().size()); // AMEX - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 376454057005914"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 376454057005914"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 346119657106278."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 346119657106278."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.CREDIT_CARD)); // MASTERCARD - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 5567408136464000"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 5567408136464000"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 5100170632668000."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 5100170632668000."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); // DISCOVER - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 6011485579364000"); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 6011485579364000"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); - filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the payment method is 6011792597726000."); + filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the payment method is 6011792597726000."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 38, FilterType.CREDIT_CARD)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CurrencyFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CurrencyFilterTest.java index 6b8ae00e..1e271817 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CurrencyFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CurrencyFilterTest.java @@ -46,7 +46,7 @@ public void filter1() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $35.53 ."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $35.53 ."); showSpans(filterResult.getSpans()); @@ -68,7 +68,7 @@ public void filter2() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $35.53."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $35.53."); showSpans(filterResult.getSpans()); @@ -90,7 +90,7 @@ public void filter3() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $35.00."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $35.00."); showSpans(filterResult.getSpans()); @@ -112,7 +112,7 @@ public void filter4() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $3.00."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $3.00."); showSpans(filterResult.getSpans()); @@ -134,7 +134,7 @@ public void filter5() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $.50."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $.50."); showSpans(filterResult.getSpans()); @@ -159,7 +159,7 @@ public void filter6() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $50."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $50."); showSpans(filterResult.getSpans()); @@ -184,7 +184,7 @@ public void filter7() throws Exception { final CurrencyFilter filter = new CurrencyFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the drug cost is $.50."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the drug cost is $.50."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CustomDictionaryFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CustomDictionaryFilterTest.java index 40ce3c1f..9a3cf103 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CustomDictionaryFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/CustomDictionaryFilterTest.java @@ -53,7 +53,7 @@ public void filterDictionaryExactMatch() throws Exception { final Set names = new HashSet<>(Arrays.asList("george", "ted", "bill", "john")); final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, SensitivityLevel.LOW, names, false, "names", 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"He lived with Bill in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"He lived with Bill in California."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -75,7 +75,7 @@ public void filterDictionaryNoMatch() throws Exception { final Set names = new HashSet<>(Arrays.asList("george", "ted", "bill", "john")); final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.CUSTOM_DICTIONARY, filterConfiguration, SensitivityLevel.LOW, names, false, "names", 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"He lived with Sam in California."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"He lived with Sam in California."); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DateFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DateFilterTest.java index 8ea5455b..0aedc2f7 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DateFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DateFilterTest.java @@ -51,7 +51,7 @@ public void filterDate1() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "May 22, 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "May 22, 1999"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 12, FilterType.DATE)); @@ -64,7 +64,7 @@ public void filterDate2() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "13-06-31"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "13-06-31"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 8, FilterType.DATE)); @@ -75,7 +75,7 @@ public void filterDate3() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "2205-02-31"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "2205-02-31"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 10, FilterType.DATE)); @@ -85,7 +85,7 @@ public void filterDate3() throws Exception { public void filterDate4() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-31-2019"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-31-2019"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 10, FilterType.DATE)); @@ -97,7 +97,7 @@ public void filterDate5() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-31-19"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-31-19"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 8, FilterType.DATE)); @@ -108,7 +108,7 @@ public void filterDate6() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "2-8-2019"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "2-8-2019"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 8, FilterType.DATE)); @@ -119,7 +119,7 @@ public void filterDate7() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "2-15-2019"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "2-15-2019"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 9, FilterType.DATE)); @@ -130,7 +130,7 @@ public void filterDate8() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "January 2012"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "January 2012"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 12, FilterType.DATE)); @@ -141,7 +141,7 @@ public void filterDate9() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "December 2015"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "December 2015"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 13, FilterType.DATE)); @@ -152,7 +152,7 @@ public void filterDate10() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 13, FilterType.DATE)); @@ -163,7 +163,7 @@ public void filterDate11() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "april 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "april 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 10, FilterType.DATE)); @@ -174,7 +174,7 @@ public void filterDate12() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "12-05-2014"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "12-05-2014"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 10, FilterType.DATE)); @@ -185,7 +185,7 @@ public void filterDate13() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 22, 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 22, 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 17, FilterType.DATE)); @@ -196,7 +196,7 @@ public void filterDate14() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 22nd, 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 22nd, 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 19, FilterType.DATE)); @@ -207,7 +207,7 @@ public void filterDate15() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 22 nd, 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 22 nd, 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 20, FilterType.DATE)); @@ -218,7 +218,7 @@ public void filterDate16() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 22nd"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 22nd"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 13, FilterType.DATE)); @@ -229,7 +229,7 @@ public void filterDate17() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "May 1 st"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "May 1 st"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 8, FilterType.DATE)); @@ -240,7 +240,7 @@ public void filterDate18() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "June 13th"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "June 13th"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 9, FilterType.DATE)); @@ -251,7 +251,7 @@ public void filterDate19() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "November 2, 1999"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "November 2, 1999"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 16, FilterType.DATE)); @@ -262,7 +262,7 @@ public void filterDate20() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "May 1st"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "May 1st"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 7, FilterType.DATE)); @@ -273,7 +273,7 @@ public void filterDate21() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "December 4th"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "December 4th"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 12, FilterType.DATE)); @@ -284,7 +284,7 @@ public void filterDate22() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-31-19@12:00"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-31-19@12:00"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 8, FilterType.DATE)); @@ -295,7 +295,7 @@ public void filterDate23() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-31-19@12:00"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-31-19@12:00"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -306,7 +306,7 @@ public void filterDate24() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-35-19@12:00"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-35-19@12:00"); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -316,7 +316,7 @@ public void filterDate25() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "02-15-19"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "02-15-19"); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -326,7 +326,7 @@ public void filterDate26() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may be on top of that."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -336,7 +336,7 @@ public void filterDate27() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -346,7 +346,7 @@ public void filterDate28() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15, 2020 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15, 2020 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -356,7 +356,7 @@ public void filterDate29() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 2020 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 2020 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -366,7 +366,7 @@ public void filterDate30() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 19 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 15 19 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -376,7 +376,7 @@ public void filterDate31() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 5 19 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this may 5 19 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -386,7 +386,7 @@ public void filterDate32() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this June 21, 2020 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this June 21, 2020 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals("June 21, 2020", filterResult.getSpans().get(0).getText()); @@ -399,7 +399,7 @@ public void filterDate33() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this 09-2021 be on top of that."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The good news is everywhere we go it is that way but this 09-2021 be on top of that."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals("09-2021", filterResult.getSpans().get(0).getText()); @@ -422,14 +422,14 @@ public void filterDate34() throws Exception { final DateFilter filter = new DateFilter(filterConfiguration, false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Owns and drives his own vehicle but states he has not driven his car since last October 2009."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Owns and drives his own vehicle but states he has not driven his car since last October 2009."); LOGGER.info(filterResult.getSpans().get(0).getReplacement()); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); - // This can be "12 years" or "13 years" depending on when the test is run. - Assertions.assertTrue(filterResult.getSpans().get(0).getReplacement().startsWith("12 years") || filterResult.getSpans().get(0).getReplacement().startsWith("13 years")); + // This value can differ depending on when the test is run. + Assertions.assertTrue(filterResult.getSpans().get(0).getReplacement().startsWith("14 years") || filterResult.getSpans().get(0).getReplacement().startsWith("13 years")); } @@ -438,7 +438,7 @@ public void filterDate35() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Observation: 91-100% strong nuclear staining"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Observation: 91-100% strong nuclear staining"); Assertions.assertEquals(0, filterResult.getSpans().size()); showSpans(filterResult.getSpans()); @@ -450,7 +450,7 @@ public void filterDate36() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary petition on behalf of FKAAHS, Inc. fka Aire"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary petition on behalf of FKAAHS, Inc. fka Aire"); showSpans(filterResult.getSpans()); @@ -468,7 +468,7 @@ public void filterDate37() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary petition on behalf of FKAAHS, Inc. fka Aire"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary petition on behalf of FKAAHS, Inc. fka Aire"); showSpans(filterResult.getSpans()); @@ -484,7 +484,7 @@ public void filterDate38() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), false, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Case No. 12-12110 K"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Case No. 12-12110 K"); showSpans(filterResult.getSpans()); @@ -500,7 +500,7 @@ public void filterDate39() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Case No. 12-12110 K"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Case No. 12-12110 K"); showSpans(filterResult.getSpans()); @@ -513,7 +513,7 @@ public void filterDate40() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Case 1-20-01023-MJK"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Case 1-20-01023-MJK"); showSpans(filterResult.getSpans()); @@ -526,7 +526,7 @@ public void filterDate41() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "That on July 3, 2012 an involuntary"); showSpans(filterResult.getSpans()); @@ -542,7 +542,7 @@ public void filterDate42() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Entered 06/16/20 11:55:37,"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Entered 06/16/20 11:55:37,"); showSpans(filterResult.getSpans()); @@ -558,7 +558,7 @@ public void filterDate43() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Case 1-20-01023-MJK, Doc 1, Filed 06/16/20, Entered 06/16/20 11:55:37,"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Case 1-20-01023-MJK, Doc 1, Filed 06/16/20, Entered 06/16/20 11:55:37,"); showSpans(filterResult.getSpans()); @@ -577,7 +577,7 @@ public void filterDate44() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "On August 22, 2012, Plaintiff, Wendy J. Christophersen, was appointed as interim"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "On August 22, 2012, Plaintiff, Wendy J. Christophersen, was appointed as interim"); showSpans(filterResult.getSpans()); @@ -595,7 +595,7 @@ public void filterDate45() throws Exception { final DateFilter filter = new DateFilter(buildFilterConfiguration(), true, DateSpanValidator.getInstance()); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "On Aug. 31, 2020, Plaintiff, Wendy J. Christophersen, was appointed as interim"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "On Aug. 31, 2020, Plaintiff, Wendy J. Christophersen, was appointed as interim"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DriversLicenseFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DriversLicenseFilterTest.java index 1174aa83..2b4154e4 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DriversLicenseFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/DriversLicenseFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final DriversLicenseFilter filter = new DriversLicenseFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is 123456789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is 123456789."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/EmailAddressFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/EmailAddressFilterTest.java index bbf1acdd..45c83f3a 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/EmailAddressFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/EmailAddressFilterTest.java @@ -45,7 +45,7 @@ public void filterEmail() throws Exception { final EmailAddressFilter filter = new EmailAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "my email is none@none.com."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "my email is none@none.com."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 25, FilterType.EMAIL_ADDRESS)); Assertions.assertEquals("none@none.com", filterResult.getSpans().get(0).getText()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FilterTest.java index cf5e5a5d..67468deb 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FilterTest.java @@ -51,7 +51,7 @@ public void window0() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); showSpans(filterResult.getSpans()); @@ -82,7 +82,7 @@ public void window1() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); showSpans(filterResult.getSpans()); @@ -114,7 +114,7 @@ public void window2() throws Exception { final AgeFilter filter = new AgeFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "this is a first sentence. the patient is 3.5 years old and he's cool. this is a surrounding sentence."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FirstNameFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FirstNameFilterTest.java index ba723f50..47cc6dd2 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FirstNameFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/FirstNameFilterTest.java @@ -59,7 +59,7 @@ public void filterLow() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"John"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"John"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -77,7 +77,7 @@ public void filterMedium1() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Michel had eye cancer"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Michel had eye cancer"); showSpans(filterResult.getSpans()); Assertions.assertEquals(3, filterResult.getSpans().size()); @@ -95,7 +95,7 @@ public void filterMedium2() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Jennifer had eye cancer"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Jennifer had eye cancer"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -113,7 +113,7 @@ public void filterHigh() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Sandra in Washington"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Sandra in Washington"); showSpans(filterResult.getSpans()); Assertions.assertEquals(3, filterResult.getSpans().size()); @@ -131,7 +131,7 @@ public void filter1() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Melissa"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Melissa"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -150,7 +150,7 @@ public void filter2() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"que"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"que"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -168,7 +168,7 @@ public void filter3() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"dat"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"dat"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -186,7 +186,7 @@ public void filter4() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"joie"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"joie"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -204,7 +204,7 @@ public void filter5() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"John"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"John"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -222,7 +222,7 @@ public void filter6() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.FIRST_NAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Smith,Melissa A,MD"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Smith,Melissa A,MD"); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalAbbreviationFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalAbbreviationFilterTest.java index 161fc349..2f5f9bad 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalAbbreviationFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalAbbreviationFilterTest.java @@ -86,7 +86,7 @@ public void filter1() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.HOSPITAL_ABBREVIATION, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Went to WMC"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Went to WMC"); showSpans(filterResult.getSpans()); Assertions.assertEquals(4, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalFilterTest.java index 876cb524..fb607f0c 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/HospitalFilterTest.java @@ -59,7 +59,7 @@ public void filter1() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.HOSPITAL, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Wyoming Medical Center"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Wyoming Medical Center"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals("wyoming medical center", filterResult.getSpans().get(0).getText()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IbanCodeFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IbanCodeFilterTest.java index ef5a812a..537ae097 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IbanCodeFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IbanCodeFilterTest.java @@ -54,7 +54,7 @@ public void filter1() throws Exception { final Filter filter = getFilter(true, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "bank code of GB33BUKB20201555555555 ok?"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "bank code of GB33BUKB20201555555555 ok?"); showSpans(filterResult.getSpans()); @@ -70,7 +70,7 @@ public void filter2() throws Exception { final Filter filter = getFilter(false, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "bank code of GB15MIDL40051512345678 ok?"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "bank code of GB15MIDL40051512345678 ok?"); showSpans(filterResult.getSpans()); @@ -86,7 +86,7 @@ public void filter3() throws Exception { final Filter filter = getFilter(true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "bank code of GB15 MIDL 4005 1512 3456 78 ok?"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "bank code of GB15 MIDL 4005 1512 3456 78 ok?"); showSpans(filterResult.getSpans()); @@ -102,7 +102,7 @@ public void filter4() throws Exception { final Filter filter = getFilter(true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "bank code of GB15 MIDL 4005 1512 3456 zz ok?"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "bank code of GB15 MIDL 4005 1512 3456 zz ok?"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IdentifierFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IdentifierFilterTest.java index 2c388761..a7ce4ed8 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IdentifierFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IdentifierFilterTest.java @@ -49,7 +49,7 @@ public void filterId1() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is AB4736021 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is AB4736021 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,19, FilterType.IDENTIFIER)); @@ -69,7 +69,7 @@ public void filterId2() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is AB473-6021 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is AB473-6021 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,20, FilterType.IDENTIFIER)); @@ -88,7 +88,7 @@ public void filterId3() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 473-6AB021 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 473-6AB021 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,20, FilterType.IDENTIFIER)); @@ -107,7 +107,7 @@ public void filterId4() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is AB473-6021 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is AB473-6021 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,20, FilterType.IDENTIFIER)); @@ -126,7 +126,7 @@ public void filterId5() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 473-6AB021 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 473-6AB021 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,20, FilterType.IDENTIFIER)); @@ -145,7 +145,7 @@ public void filterId6() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 123-45-6789 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 123-45-6789 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,21, FilterType.IDENTIFIER)); @@ -164,7 +164,7 @@ public void filterId7() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived at 90210. Patient id 00076A and 93821A. He is on biotin. Diagnosed with A01000."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived at 90210. Patient id 00076A and 93821A. He is on biotin. Diagnosed with A01000."); Assertions.assertEquals(4, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 48, 59, FilterType.IDENTIFIER)); @@ -186,7 +186,7 @@ public void filterId8() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 000-00-00-00 ABC123 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 000-00-00-00 ABC123 in california."); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10, 22, FilterType.IDENTIFIER)); @@ -206,7 +206,7 @@ public void filterId9() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is AZ12 ABC1234/123ABC4 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is AZ12 ABC1234/123ABC4 in california."); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 15, 22, FilterType.IDENTIFIER)); @@ -226,7 +226,7 @@ public void filterId10() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is H3SNPUHYEE7JD3H in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is H3SNPUHYEE7JD3H in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,25, FilterType.IDENTIFIER)); @@ -245,7 +245,7 @@ public void filterId11() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 86637729 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 86637729 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,18, FilterType.IDENTIFIER)); @@ -264,7 +264,7 @@ public void filterId12() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 33778376 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 33778376 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,18, FilterType.IDENTIFIER)); @@ -283,7 +283,7 @@ public void filterId13() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", "\\b[A-Z]{4,}\\b", true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is ABCD."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is ABCD."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,14, FilterType.IDENTIFIER)); @@ -302,7 +302,7 @@ public void filterId14() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the id is 123456."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the id is 123456."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10,16, FilterType.IDENTIFIER)); @@ -321,7 +321,7 @@ public void filterId15() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", Identifier.DEFAULT_IDENTIFIER_REGEX, true, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "John Smith, patient ID A203493, was seen on February 18."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "John Smith, patient ID A203493, was seen on February 18."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -341,7 +341,7 @@ public void filterId16() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", "\\b\\d{3,8}\\b", false, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "The ID is 123456."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "The ID is 123456."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -361,7 +361,7 @@ public void filterId17() throws Exception { final IdentifierFilter filter = new IdentifierFilter(filterConfiguration, "name", "(?i)([^Application Name])(.*)$", false, 0); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Application Name John Smith"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Application Name John Smith"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IpAddressFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IpAddressFilterTest.java index 2795fba6..e2d37d1f 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IpAddressFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/IpAddressFilterTest.java @@ -45,7 +45,7 @@ public void filterIpv41() throws Exception { final IpAddressFilter filter = new IpAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ip is 192.168.1.101."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ip is 192.168.1.101."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10, 23, FilterType.IP_ADDRESS)); @@ -65,7 +65,7 @@ public void filterIpv61() throws Exception { final IpAddressFilter filter = new IpAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ip is 1::"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ip is 1::"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 10, 13, FilterType.IP_ADDRESS)); @@ -84,7 +84,7 @@ public void filterIpv62() throws Exception { final IpAddressFilter filter = new IpAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ip is 2001:0db8:85a3:0000:0000:8a2e:0370:7334"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ip is 2001:0db8:85a3:0000:0000:8a2e:0370:7334"); // Finds duplicate spans. Duplicates/overlapping will be removed by the service prior to returning. Assertions.assertEquals(2, filterResult.getSpans().size()); @@ -105,7 +105,7 @@ public void filterIpv63() throws Exception { final IpAddressFilter filter = new IpAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ip is fe80::0202:B3FF:FE1E:8329"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ip is fe80::0202:B3FF:FE1E:8329"); // Finds duplicate spans. Duplicates/overlapping will be removed by the service prior to returning. Assertions.assertEquals(2, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/MacAddressFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/MacAddressFilterTest.java index 34581b76..454fc0bb 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/MacAddressFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/MacAddressFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final MacAddressFilter filter = new MacAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the mac is 00-14-22-04-25-37."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the mac is 00-14-22-04-25-37."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.MAC_ADDRESS)); @@ -65,7 +65,7 @@ public void filter2() throws Exception { final MacAddressFilter filter = new MacAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the mac is 00:14:22:04:25:37."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the mac is 00:14:22:04:25:37."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.MAC_ADDRESS)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PassportNumberFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PassportNumberFilterTest.java index d456b4cc..86b7654b 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PassportNumberFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PassportNumberFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final PassportNumberFilter filter = new PassportNumberFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the passport number is 036001231."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the passport number is 036001231."); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV2FilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV2FilterTest.java index 41543aef..1ff0d67d 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV2FilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV2FilterTest.java @@ -71,7 +71,7 @@ public void filter1() throws Exception { metricsService, thresholds); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 17, FilterType.PERSON)); @@ -104,7 +104,7 @@ public void filter2() throws Exception { metricsService, thresholds); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived at 90210. The name 456 should be filtered. Jeff Smith should be ignored."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived at 90210. The name 456 should be filtered. Jeff Smith should be ignored."); showSpans(filterResult.getSpans()); @@ -146,7 +146,7 @@ public void filter3() throws Exception { final String input = "In recent days, healthcare facilities across the nation have again begun to buckle under spiking infection rates. Last week, some local hospitals temporarily postponed scheduled surgeries that require an inpatient stay following an operation, and the trauma center at Harbor-UCLA Medical Center closed for hours because of a blood shortage - a step it hadn't taken in over three decades. A staff shortage at some local ambulance companies further complicated the situation.The virus has spread so fast since the arrival of the Omicron variant that it could take just about a week for California to tally a million new cases. It was only on Jan. 10 that California surpassed 6 million total reported coronavirus cases in the nearly two years since the start of the pandemic, according to data released by state health officials. Even during last winter's surge, it took three weeks to accumulate a million new cases, with the state peaking at 46,000 new infections a day. \"On this national holiday where we celebrate the life and legacy of Dr. Martin Luther King, we remember his deep commitment to health equity,\" said L.A. County Public Health Director Barbara Ferrer Ferrer. \"As Reverend King memorably said, \"Of all the forms of inequality, injustice in health is the most shocking and the most inhuman because it often results in physical death.\""; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, input); showSpans(filterResult.getSpans()); @@ -184,7 +184,7 @@ public void filter4() throws Exception { final String input = "IN THE UNITED STATES DISTRICT COURT \nEASTERN DISTRICT OF ARKANSAS \nWESTERN DIVISION \nJAMES EDWARD SMITH, \nafk/a James Edward Bridges, \nADC#103093 \nv. No. 4:14-cv-455-DPM \nPLAINTIFF \nCHARLES A. SMITH; \nMARY ANN CONLEY, \nafk/a Mary Ann Smith; and \nROBERT CASTILLOW DEFENDANTS \nORDER \nJames Smith's prose complaint must be dismissed without prejudice. \nHe hasn't paid the filing fee, moved to proceed in forma pauperis, or provided \nproof of service on any defendant. FED. R. CIV. P. 4(I); Local Rule 5.5(c)(2). \nSo Ordered. \nD.P. Marshall Jr. \nUnited States District Judge \nCase 4:14-cv-00455-DPM Document 2 Filed 12/09/14 Page 1 of 1\n"; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, input); showSpans(filterResult.getSpans()); @@ -218,7 +218,7 @@ public void filter5() throws Exception { final String input = "Plaintiff, Wendy J. Christophersen, trustee in the estate of the above-named debtor, complains of Defendant and shows the court as follows:"; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, input); showSpans(filterResult.getSpans()); @@ -254,7 +254,7 @@ public void filter6() throws Exception { final String input = "Plaintiff, Wendy J. Christophersen, trustee in the estate of the above-named debtor,"; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, input); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV3FilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV3FilterTest.java index 854ccdf6..dbb3116f 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV3FilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PersonsV3FilterTest.java @@ -65,7 +65,7 @@ public void filter1() throws Exception { metricsService, thresholds); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington and Abraham Lincoln were presidents."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington and Abraham Lincoln were presidents."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.PERSON)); @@ -96,7 +96,7 @@ public void filter2() throws Exception { metricsService, thresholds); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington and Abraham Lincoln were presidents."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington and Abraham Lincoln were presidents."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 22, 37, FilterType.PERSON)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberExtensionFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberExtensionFilterTest.java index c5151277..fa58ef52 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberExtensionFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberExtensionFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final PhoneNumberExtensionFilter filter = new PhoneNumberExtensionFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "he is at x123"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "he is at x123"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -66,7 +66,7 @@ public void filter2() throws Exception { final PhoneNumberExtensionFilter filter = new PhoneNumberExtensionFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "his phone number was +1 151-841-2829 x416."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "his phone number was +1 151-841-2829 x416."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberFilterTest.java index 1421dac4..65080523 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhoneNumberFilterTest.java @@ -49,7 +49,7 @@ public void filterPhone1() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is (123) 456-7890."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is (123) 456-7890."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 14, 28, FilterType.PHONE_NUMBER)); Assertions.assertEquals("(123) 456-7890", filterResult.getSpans().get(0).getText()); @@ -69,7 +69,7 @@ public void filterPhone2() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is (123) 456-7890 and (123) 456-7890."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is (123) 456-7890 and (123) 456-7890."); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 14, 28, FilterType.PHONE_NUMBER)); Assertions.assertEquals(0.95, filterResult.getSpans().get(0).getConfidence()); @@ -90,7 +90,7 @@ public void filterPhone3() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is 123-456-7890."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is 123-456-7890."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 14, 26, FilterType.PHONE_NUMBER)); Assertions.assertEquals(0.95, filterResult.getSpans().get(0).getConfidence()); @@ -109,7 +109,7 @@ public void filterPhone4() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is 123-456-7890 and he was ok."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is 123-456-7890 and he was ok."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 14, 26, FilterType.PHONE_NUMBER)); @@ -127,7 +127,7 @@ public void filterPhone5() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is ( 800 ) 123-4567 and he was ok."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is ( 800 ) 123-4567 and he was ok."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 14, 30, FilterType.PHONE_NUMBER)); Assertions.assertEquals(0.75, filterResult.getSpans().get(0).getConfidence()); @@ -146,7 +146,7 @@ public void filterPhone6() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is (800) 123-4567 x532 and he was ok."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is (800) 123-4567 x532 and he was ok."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -167,7 +167,7 @@ public void filterPhone7() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the number is (800) 123-4567x532 and he was ok."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the number is (800) 123-4567x532 and he was ok."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -188,7 +188,7 @@ public void filterPhone8() throws Exception { final PhoneNumberRulesFilter filter = new PhoneNumberRulesFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "7 64116-3220"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "7 64116-3220"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhysicianNameFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhysicianNameFilterTest.java index bf41ded5..b48ed4a7 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhysicianNameFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/PhysicianNameFilterTest.java @@ -49,7 +49,7 @@ public void physicianNameTestPreNominal1() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Doctor Smith was the attending physician."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Doctor Smith was the attending physician."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -70,7 +70,7 @@ public void physicianNameTestPreNominal2() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Doctor James Smith"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Doctor James Smith"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -91,7 +91,7 @@ public void physicianNameTestPostNominal1() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "John Smith, MD"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "John Smith, MD"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -112,7 +112,7 @@ public void physicianNameTestPostNominal2() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "attending physician was John Smith, MD"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "attending physician was John Smith, MD"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -133,7 +133,7 @@ public void physicianNameTestPostNominal3() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "was John J. van Smith, MD"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "was John J. van Smith, MD"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -154,7 +154,7 @@ public void physicianNameTestPostNominal4() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Smith,James D,MD -General Surgery"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Smith,James D,MD -General Surgery"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -175,7 +175,7 @@ public void physicianNameTestPostNominal5() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Smith,James )D,MD -General Surgery"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Smith,James )D,MD -General Surgery"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -196,7 +196,7 @@ public void physicianNameTestPostNominal6() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "1.0 cm in outside diameter pink tan everted"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "1.0 cm in outside diameter pink tan everted"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -215,7 +215,7 @@ public void physicianNameTestPostNominal7() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "1.0 cm"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "1.0 cm"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -234,7 +234,7 @@ public void physicianNameTestPostNominal8() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Ahu,Amanda D,MD -General Surgery"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Ahu,Amanda D,MD -General Surgery"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -255,7 +255,7 @@ public void physicianNameTestPostNominal9() throws Exception { final PhysicianNameFilter filter = new PhysicianNameFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Johnns,Melinda S,MD - 1/2/2018 11:54 CST 1/2/2018 12:46 CST"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Johnns,Melinda S,MD - 1/2/2018 11:54 CST 1/2/2018 12:46 CST"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SectionFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SectionFilterTest.java index 4a566955..c3d925c4 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SectionFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SectionFilterTest.java @@ -48,7 +48,7 @@ public void filterSection1() throws Exception { final SectionFilter filter = new SectionFilter(filterConfiguration, startPattern, endPattern); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "This is some test. BEGIN-REDACT This text should be redacted. END-REDACT This is outside the text."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "This is some test. BEGIN-REDACT This text should be redacted. END-REDACT This is outside the text."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 19, 72, FilterType.SECTION)); @@ -71,7 +71,7 @@ public void filterSection2() throws Exception { final SectionFilter filter = new SectionFilter(filterConfiguration, startPattern, endPattern); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "This is some test. BEGIN-REDACT This text should be redacted. This is outside the text."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "This is some test. BEGIN-REDACT This text should be redacted. This is outside the text."); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -92,7 +92,7 @@ public void filterSection3() throws Exception { final SectionFilter filter = new SectionFilter(filterConfiguration, startPattern, endPattern); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "BEGIN-REDACT This text should be redacted. END-REDACT This is outside the text."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "BEGIN-REDACT This text should be redacted. END-REDACT This is outside the text."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 53, FilterType.SECTION)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SsnFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SsnFilterTest.java index 054e5c97..321d16b1 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SsnFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SsnFilterTest.java @@ -45,7 +45,7 @@ public void filterSsn1() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123-45-6789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123-45-6789."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 22, FilterType.SSN)); Assertions.assertEquals("123-45-6789", filterResult.getSpans().get(0).getText()); @@ -64,7 +64,7 @@ public void filterSsn2() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123456789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123456789."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 20, FilterType.SSN)); @@ -82,7 +82,7 @@ public void filterSsn3() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123 45 6789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123 45 6789."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 22, FilterType.SSN)); @@ -100,7 +100,7 @@ public void filterSsn4() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123 45 6789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123 45 6789."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 22, FilterType.SSN)); @@ -118,7 +118,7 @@ public void filterSsn5() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123 454 6789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123 454 6789."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -135,7 +135,7 @@ public void filterSsn6() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 123 4f 6789."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 123 4f 6789."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -152,7 +152,7 @@ public void filterSsn7() throws Exception { final SsnFilter filter = new SsnFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the ssn is 11-1234567."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the ssn is 11-1234567."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 21, FilterType.SSN)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateAbbreviationFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateAbbreviationFilterTest.java index 9aa4512d..299c7731 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateAbbreviationFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateAbbreviationFilterTest.java @@ -50,7 +50,7 @@ public void filter1() throws Exception { final StateAbbreviationFilter filter = new StateAbbreviationFilter(filterConfiguration); final String input = "The patient is from WV."; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "docid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "docid", PIECE, input); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals(20, filterResult.getSpans().get(0).getCharacterStart()); @@ -73,7 +73,7 @@ public void filter2() throws Exception { final StateAbbreviationFilter filter = new StateAbbreviationFilter(filterConfiguration); final String input = "The patient is from wv."; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "docid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "docid", PIECE, input); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals(20, filterResult.getSpans().get(0).getCharacterStart()); @@ -95,7 +95,7 @@ public void filter3() throws Exception { final StateAbbreviationFilter filter = new StateAbbreviationFilter(filterConfiguration); final String input = "Patients from WV and MD."; - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "docid", PIECE, input); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "docid", PIECE, input); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateFilterTest.java index 77c4994c..c2bd6f89 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StateFilterTest.java @@ -59,7 +59,7 @@ public void filterStatesLow() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_STATE, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"Lived in Washington"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"Lived in Washington"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertEquals("washington", filterResult.getSpans().get(0).getText()); @@ -77,7 +77,7 @@ public void filterStatesMedium() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_STATE, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Wshington"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Wshington"); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -94,7 +94,7 @@ public void filterStatesHigh() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.LOCATION_STATE, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Wasinton"); + FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Wasinton"); Assertions.assertEquals(1, filterResult.getSpans().size()); } diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StreetAddressFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StreetAddressFilterTest.java index c5e38d38..b2819ac1 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StreetAddressFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/StreetAddressFilterTest.java @@ -45,7 +45,7 @@ public void filter1() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "lived at 100 Main St"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "lived at 100 Main St"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 9, 20, FilterType.STREET_ADDRESS)); @@ -64,7 +64,7 @@ public void filter2() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "lived at 100 S Main St"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "lived at 100 S Main St"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 9, 22, FilterType.STREET_ADDRESS)); @@ -83,7 +83,7 @@ public void filter3() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "lived at 100 South Main St"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "lived at 100 South Main St"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 9, 26, FilterType.STREET_ADDRESS)); @@ -102,7 +102,7 @@ public void filter4() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "lived at 1000 Main Street"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "lived at 1000 Main Street"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 9, 25, FilterType.STREET_ADDRESS)); @@ -121,7 +121,7 @@ public void filter5() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "North 2800 Clay Edwards Drive"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "North 2800 Clay Edwards Drive"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 6, 29, FilterType.STREET_ADDRESS)); @@ -140,7 +140,7 @@ public void filter6() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "14 Southampton St."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "14 Southampton St."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 18, FilterType.STREET_ADDRESS)); @@ -159,7 +159,7 @@ public void filter7() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "22 Newport Drive"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "22 Newport Drive"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 16, FilterType.STREET_ADDRESS)); @@ -178,7 +178,7 @@ public void filter8() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "78 Glendale Street"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "78 Glendale Street"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 18, FilterType.STREET_ADDRESS)); @@ -197,7 +197,7 @@ public void filter9() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "6 Berkshire Court"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "6 Berkshire Court"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 17, FilterType.STREET_ADDRESS)); @@ -216,7 +216,7 @@ public void filter10() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "291 North Pawnee Ave."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "291 North Pawnee Ave."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 21, FilterType.STREET_ADDRESS)); @@ -235,7 +235,7 @@ public void filter11() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "468 William Street"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "468 William Street"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 18, FilterType.STREET_ADDRESS)); @@ -254,7 +254,7 @@ public void filter12() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "291 6th Dr."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "291 6th Dr."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 11, FilterType.STREET_ADDRESS)); @@ -273,7 +273,7 @@ public void filter13() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "9444 Heritage St."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "9444 Heritage St."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 17, FilterType.STREET_ADDRESS)); @@ -292,7 +292,7 @@ public void filter14() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "70 Birchpond Street"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "70 Birchpond Street"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 19, FilterType.STREET_ADDRESS)); @@ -311,7 +311,7 @@ public void filter15() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "656 S. Inverness St."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "656 S. Inverness St."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 20, FilterType.STREET_ADDRESS)); @@ -330,7 +330,7 @@ public void filter16() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "9142 Arlington Court"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "9142 Arlington Court"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 20, FilterType.STREET_ADDRESS)); @@ -349,7 +349,7 @@ public void filter17() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 16, FilterType.STREET_ADDRESS)); @@ -368,7 +368,7 @@ public void filter18() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 15, FilterType.STREET_ADDRESS)); @@ -387,7 +387,7 @@ public void filter19() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct ste 2"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct ste 2"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 21, FilterType.STREET_ADDRESS)); @@ -406,7 +406,7 @@ public void filter20() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct apartment 222"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct apartment 222"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 29, FilterType.STREET_ADDRESS)); @@ -425,7 +425,7 @@ public void filter21() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct apt 222"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct apt 222"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 23, FilterType.STREET_ADDRESS)); @@ -444,7 +444,7 @@ public void filter22() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "4 Devonshire Ct apt 222 anywhere"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "4 Devonshire Ct apt 222 anywhere"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 0, 23, FilterType.STREET_ADDRESS)); @@ -463,7 +463,7 @@ public void filter23() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "address is 9444 Heritage St. over there"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "address is 9444 Heritage St. over there"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.STREET_ADDRESS)); @@ -482,7 +482,7 @@ public void filter24() throws Exception { final StreetAddressFilter filter = new StreetAddressFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "address is 9444 Heritage St. apt 2 over there"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "address is 9444 Heritage St. apt 2 over there"); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 34, FilterType.STREET_ADDRESS)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SurnameFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SurnameFilterTest.java index afde8a08..720e1590 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SurnameFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/SurnameFilterTest.java @@ -59,7 +59,7 @@ public void filter1() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Wshington"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Wshington"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -77,7 +77,7 @@ public void filter2() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.MEDIUM, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Lived in Wshington"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Lived in Wshington"); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); @@ -95,7 +95,7 @@ public void filter3() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.HIGH, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Jones"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Jones"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -113,7 +113,7 @@ public void filter4() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "date"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "date"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -131,7 +131,7 @@ public void filter5() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "Jones"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "Jones"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); @@ -149,7 +149,7 @@ public void filter6() throws Exception { final LuceneDictionaryFilter filter = new LuceneDictionaryFilter(FilterType.SURNAME, filterConfiguration, INDEX_DIRECTORY, SensitivityLevel.LOW, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "from"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "from"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/TrackingNumberFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/TrackingNumberFilterTest.java index 9838b505..53c3d24d 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/TrackingNumberFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/TrackingNumberFilterTest.java @@ -46,7 +46,7 @@ public void filter0() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 1Z9YF1280343418566"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 1Z9YF1280343418566"); showSpans(filterResult.getSpans()); @@ -70,7 +70,7 @@ public void filter1() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); showSpans(filterResult.getSpans()); @@ -95,7 +95,7 @@ public void filter2() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 9400 1000 0000 0000 0000"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 9400 1000 0000 0000 0000"); showSpans(filterResult.getSpans()); @@ -119,7 +119,7 @@ public void filter3() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 4204319935009201990138501144099814"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 4204319935009201990138501144099814"); showSpans(filterResult.getSpans()); @@ -143,7 +143,7 @@ public void filter4() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 420431993500920199013850114409"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 420431993500920199013850114409"); showSpans(filterResult.getSpans()); @@ -167,7 +167,7 @@ public void filter5() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 4204319935009201990138501144"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 4204319935009201990138501144"); showSpans(filterResult.getSpans()); @@ -191,7 +191,7 @@ public void filter6() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); showSpans(filterResult.getSpans()); @@ -216,7 +216,7 @@ public void filter7() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, false, false, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); showSpans(filterResult.getSpans()); @@ -241,7 +241,7 @@ public void filter8() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, false, false, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 42043199350092019901385011"); showSpans(filterResult.getSpans()); @@ -261,7 +261,7 @@ public void filter9() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, false, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); showSpans(filterResult.getSpans()); @@ -285,7 +285,7 @@ public void filter10() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); showSpans(filterResult.getSpans()); @@ -309,7 +309,7 @@ public void filter11() throws Exception { final TrackingNumberFilter filter = new TrackingNumberFilter(filterConfiguration, true, true, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the tracking number is 9400100000000000000000"); showSpans(filterResult.getSpans()); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/UrlFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/UrlFilterTest.java index 12eed036..85982980 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/UrlFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/UrlFilterTest.java @@ -45,7 +45,7 @@ public void filterUrl1() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE,"the page is http://page.com."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE,"the page is http://page.com."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 27, FilterType.URL)); @@ -65,7 +65,7 @@ public void filterUrl2() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is myhomepage.com."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is myhomepage.com."); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -83,7 +83,7 @@ public void filterUrl3() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://myhomepage.com/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://myhomepage.com/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 51, FilterType.URL)); @@ -102,7 +102,7 @@ public void filterUrl4() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://www.myhomepage.com/folder/page.html"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://www.myhomepage.com/folder/page.html"); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 54, FilterType.URL)); @@ -121,7 +121,7 @@ public void filterUrl5() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is www.myhomepage.com/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is www.myhomepage.com/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 48, FilterType.URL)); @@ -140,7 +140,7 @@ public void filterUrl6() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is myhomepage.com."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is myhomepage.com."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 26, FilterType.URL)); @@ -159,7 +159,7 @@ public void filterUrl7() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is www.myhomepage.com:80/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is www.myhomepage.com:80/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 51, FilterType.URL)); @@ -178,7 +178,7 @@ public void filterUrl8() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://192.168.1.1:80/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://192.168.1.1:80/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 51, FilterType.URL)); @@ -197,7 +197,7 @@ public void filterUrl9() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is 192.168.1.1:80/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is 192.168.1.1:80/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 34, 43, FilterType.URL)); @@ -217,7 +217,7 @@ public void filterUrl10() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://192.168.1.1:80/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://192.168.1.1:80/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 41, 50, FilterType.URL)); @@ -237,7 +237,7 @@ public void filterUrl11() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is https://192.168.1.1:80/folder/page.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is https://192.168.1.1:80/folder/page.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 42, 51, FilterType.URL)); @@ -257,7 +257,7 @@ public void filterUrl12() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is test.ok new sentence"); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is test.ok new sentence"); showSpans(filterResult.getSpans()); Assertions.assertEquals(0, filterResult.getSpans().size()); @@ -275,7 +275,7 @@ public void filterUrl13() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://2001:0db8:85a3:0000:0000:8a2e:0370:7334/test.html."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://2001:0db8:85a3:0000:0000:8a2e:0370:7334/test.html."); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 59, 68, FilterType.URL)); @@ -295,7 +295,7 @@ public void filterUrl14() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is http://2001:0db8:85a3:0000:0000:8a2e:0370:7334/test/."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is http://2001:0db8:85a3:0000:0000:8a2e:0370:7334/test/."); showSpans(filterResult.getSpans()); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 12, 65, FilterType.URL)); @@ -314,7 +314,7 @@ public void filterUrl15() throws Exception { final UrlFilter filter = new UrlFilter(filterConfiguration, false); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the page is https://192.168.1.1:80/folder/page.html. this is a new sentence."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the page is https://192.168.1.1:80/folder/page.html. this is a new sentence."); showSpans(filterResult.getSpans()); Assertions.assertEquals(2, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 42, 51, FilterType.URL)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/VinFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/VinFilterTest.java index 789c9fbd..3be8343f 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/VinFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/VinFilterTest.java @@ -49,7 +49,7 @@ public void filterVin1() throws Exception { final VinFilter filter = new VinFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the vin is JB3BA36KXHU036784."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the vin is JB3BA36KXHU036784."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.VIN)); Assertions.assertEquals("JB3BA36KXHU036784", filterResult.getSpans().get(0).getText()); @@ -68,7 +68,7 @@ public void filterVin2() throws Exception { final VinFilter filter = new VinFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the vin is 2T2HK31U38C057399."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the vin is 2T2HK31U38C057399."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.VIN)); @@ -86,7 +86,7 @@ public void filterVin3() throws Exception { final VinFilter filter = new VinFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the vin is 11131517191011111."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the vin is 11131517191011111."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -103,7 +103,7 @@ public void filterVin4() throws Exception { final VinFilter filter = new VinFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the vin is 11131517191X11111."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the vin is 11131517191X11111."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -120,7 +120,7 @@ public void filterVin5() throws Exception { final VinFilter filter = new VinFilter(filterConfiguration); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the vin is 2t2hk31u38c057399."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the vin is 2t2hk31u38c057399."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 28, FilterType.VIN)); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/ZipCodeFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/ZipCodeFilterTest.java index 6edf4b64..d2107fb0 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/ZipCodeFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/filters/ZipCodeFilterTest.java @@ -45,7 +45,7 @@ public void filterZipCode1() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the zip is 90210."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the zip is 90210."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 16, FilterType.ZIP_CODE)); Assertions.assertEquals("90210", filterResult.getSpans().get(0).getText()); @@ -64,7 +64,7 @@ public void filterZipCode2() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the zip is 90210abd."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the zip is 90210abd."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -81,7 +81,7 @@ public void filterZipCode3() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the zip is 90210 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the zip is 90210 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 16, FilterType.ZIP_CODE)); @@ -99,7 +99,7 @@ public void filterZipCode4() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the zip is 85055 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the zip is 85055 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 16, FilterType.ZIP_CODE)); @@ -117,7 +117,7 @@ public void filterZipCode5() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "the zip is 90213-1544 in california."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "the zip is 90213-1544 in california."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 11, 21, FilterType.ZIP_CODE)); @@ -135,7 +135,7 @@ public void filterZipCode6() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 90210."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 90210."); Assertions.assertEquals(1, filterResult.getSpans().size()); Assertions.assertTrue(checkSpan(filterResult.getSpans().get(0), 76, 81, FilterType.ZIP_CODE)); @@ -154,7 +154,7 @@ public void filterZipCode7() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); // Tests whole word only. - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 9021032."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 9021032."); Assertions.assertEquals(0, filterResult.getSpans().size()); } @@ -172,7 +172,7 @@ public void filterZipCode8() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); // Tests whole word only. - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 90210-1234."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 90210-1234."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -190,7 +190,7 @@ public void filterZipCode9() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, false); // Tests without delimiter. - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 902101234."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 902101234."); Assertions.assertEquals(1, filterResult.getSpans().size()); } @@ -208,7 +208,7 @@ public void filterZipCode10() throws Exception { final ZipCodeFilter filter = new ZipCodeFilter(filterConfiguration, true); // Tests without delimiter. - final FilterResult filterResult = filter.filter(getFilterProfile(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 902101234."); + final FilterResult filterResult = filter.filter(getPolicy(), "context", "documentid", PIECE, "George Washington was president and his ssn was 123-45-6789 and he lived in 902101234."); Assertions.assertEquals(0, filterResult.getSpans().size()); } diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredPatternsFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredPatternsFilterTest.java index d77ce6b3..c5ecde73 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredPatternsFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredPatternsFilterTest.java @@ -17,7 +17,7 @@ import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.IgnoredPattern; import ai.philterd.phileas.services.postfilters.IgnoredPatternsFilter; import org.junit.jupiter.api.Assertions; @@ -36,8 +36,8 @@ public void ignored1() { ignoredPattern.setPattern("[A-Z0-9]{4}"); ignoredPattern.setName("example-id"); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setIgnoredPatterns(Arrays.asList(ignoredPattern)); + final Policy policy = new Policy(); + policy.setIgnoredPatterns(Arrays.asList(ignoredPattern)); final List spans = new LinkedList<>(); spans.add(Span.make(6, 10, FilterType.IDENTIFIER, "context", "docid", 0.80, "AB01", "*****", "", false, new String[0])); @@ -56,8 +56,8 @@ public void notIgnored1() { ignoredPattern.setPattern("[A-Z0-9]{4}"); ignoredPattern.setName("example-id"); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setIgnoredPatterns(Arrays.asList(ignoredPattern)); + final Policy policy = new Policy(); + policy.setIgnoredPatterns(Arrays.asList(ignoredPattern)); final List spans = new LinkedList<>(); spans.add(Span.make(6, 10, FilterType.IDENTIFIER, "context", "docid", 0.80, "Ab01", "*****", "", false, new String[0])); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredTermsFilterTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredTermsFilterTest.java index 0af0365d..3ae721ea 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredTermsFilterTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/postfilters/IgnoredTermsFilterTest.java @@ -17,7 +17,7 @@ import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Ignored; import ai.philterd.phileas.services.postfilters.IgnoredTermsFilter; import org.junit.jupiter.api.Assertions; @@ -37,8 +37,8 @@ public void ignored() throws IOException { final Ignored ignored = new Ignored(); ignored.setTerms(Arrays.asList("Washington", "California", "Virginia")); - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setIgnored(Arrays.asList(ignored)); + final Policy policy = new Policy(); + policy.setIgnored(Arrays.asList(ignored)); final List spans = new LinkedList<>(); spans.add(Span.make(12, 22, FilterType.LOCATION_STATE, "context", "docid", 0.80, "test", "*****", "", false, new String[0])); diff --git a/phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3FilterProfileServiceTest.java b/phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3PolicyServiceTest.java similarity index 73% rename from phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3FilterProfileServiceTest.java rename to phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3PolicyServiceTest.java index c8f68c0b..564b1515 100644 --- a/phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3FilterProfileServiceTest.java +++ b/phileas-core/src/test/java/ai/philterd/test/phileas/services/registry/S3PolicyServiceTest.java @@ -17,14 +17,14 @@ import com.google.gson.Gson; import ai.philterd.phileas.configuration.PhileasConfiguration; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Identifiers; import ai.philterd.phileas.model.profile.filters.Age; import ai.philterd.phileas.model.profile.filters.CreditCard; import ai.philterd.phileas.model.profile.filters.strategies.rules.AgeFilterStrategy; import ai.philterd.phileas.model.profile.filters.strategies.rules.CreditCardFilterStrategy; -import ai.philterd.phileas.model.services.FilterProfileService; -import ai.philterd.phileas.services.profiles.S3FilterProfileService; +import ai.philterd.phileas.model.services.PolicyService; +import ai.philterd.phileas.services.profiles.S3PolicyService; import io.findify.s3mock.S3Mock; import org.aeonbits.owner.ConfigFactory; import org.apache.commons.lang3.StringUtils; @@ -43,9 +43,9 @@ import static org.junit.jupiter.api.Assumptions.assumeFalse; -public class S3FilterProfileServiceTest { +public class S3PolicyServiceTest { - private static final Logger LOGGER = LogManager.getLogger(S3FilterProfileServiceTest.class); + private static final Logger LOGGER = LogManager.getLogger(S3PolicyServiceTest.class); private Gson gson = new Gson(); private RedisServer redisServer; @@ -139,13 +139,13 @@ public void after() { @Test public void list() throws IOException { - final FilterProfileService filterProfileService = new S3FilterProfileService(getConfiguration(), true); + final PolicyService policyService = new S3PolicyService(getConfiguration(), true); - filterProfileService.save(gson.toJson(getFilterProfile("name1"))); - filterProfileService.save(gson.toJson(getFilterProfile("name2"))); - final List names = filterProfileService.get(); + policyService.save(gson.toJson(getPolicy("name1"))); + policyService.save(gson.toJson(getPolicy("name2"))); + final List names = policyService.get(); - LOGGER.info("Found {} filter profiles", names.size()); + LOGGER.info("Found {} policies", names.size()); Assertions.assertTrue(names.size() == 2); Assertions.assertTrue(names.contains("name1")); @@ -156,12 +156,12 @@ public void list() throws IOException { @Test public void getAll() throws IOException { - final FilterProfileService filterProfileService = new S3FilterProfileService(getConfiguration(), true); + final PolicyService policyService = new S3PolicyService(getConfiguration(), true); - filterProfileService.save(gson.toJson(getFilterProfile("name1"))); - filterProfileService.save(gson.toJson(getFilterProfile("name2"))); + policyService.save(gson.toJson(getPolicy("name1"))); + policyService.save(gson.toJson(getPolicy("name2"))); - final Map all = filterProfileService.getAll(); + final Map all = policyService.getAll(); LOGGER.info("Found {} profiles", all.size()); @@ -176,13 +176,13 @@ public void save() throws IOException { final String name = "default"; - final String profile = gson.toJson(getFilterProfile(name)); + final String profile = gson.toJson(getPolicy(name)); - final FilterProfileService filterProfileService = new S3FilterProfileService(getConfiguration(), true); + final PolicyService policyService = new S3PolicyService(getConfiguration(), true); - filterProfileService.save(profile); + policyService.save(profile); - final String saved = filterProfileService.get("default"); + final String saved = policyService.get("default"); Assertions.assertNotNull(saved); Assertions.assertEquals(profile, saved); @@ -194,15 +194,15 @@ public void get() throws IOException { final String name = "default"; - final String profile = gson.toJson(getFilterProfile(name)); + final String profile = gson.toJson(getPolicy(name)); - final FilterProfileService filterProfileService = new S3FilterProfileService(getConfiguration(), true); + final PolicyService policyService = new S3PolicyService(getConfiguration(), true); - filterProfileService.save(profile); + policyService.save(profile); - final String filterProfileJson = filterProfileService.get(name); + final String policyJson = policyService.get(name); - Assertions.assertEquals(profile, filterProfileJson); + Assertions.assertEquals(profile, policyJson); } @@ -210,19 +210,19 @@ public void get() throws IOException { public void delete() throws IOException { final String name = "default"; - final String profile = gson.toJson(getFilterProfile(name)); + final String profile = gson.toJson(getPolicy(name)); - final FilterProfileService filterProfileService = new S3FilterProfileService(getConfiguration(), true); + final PolicyService policyService = new S3PolicyService(getConfiguration(), true); - filterProfileService.save(profile); + policyService.save(profile); - filterProfileService.delete(name); + policyService.delete(name); - Assertions.assertFalse(filterProfileService.getAll().containsKey(name)); + Assertions.assertFalse(policyService.getAll().containsKey(name)); } - private FilterProfile getFilterProfile(String name) { + private Policy getPolicy(String name) { AgeFilterStrategy ageFilterStrategy = new AgeFilterStrategy(); @@ -238,11 +238,11 @@ private FilterProfile getFilterProfile(String name) { identifiers.setAge(age); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName(name); - filterProfile.setIdentifiers(identifiers); + Policy policy = new Policy(); + policy.setName(name); + policy.setIdentifiers(identifiers); - return filterProfile; + return policy; } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/domain/Domain.java b/phileas-model/src/main/java/ai/philterd/phileas/model/domain/Domain.java index 51d1ff3d..281f528b 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/domain/Domain.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/domain/Domain.java @@ -22,12 +22,12 @@ /** * Represents an industry domain in which Philter can be used. - * A domain provides convenience filter profile configuration + * A domain provides convenience policy configuration * options. */ public abstract class Domain { - // These are the domain names given in the filter profile + // These are the domain names given in the policy // used to trigger the activation of the respective domain. public static final String DOMAIN_LEGAL = "legal"; public static final String DOMAIN_HEALTH = "health"; @@ -44,8 +44,8 @@ public abstract class Domain { * domain. * * These filters will only be applied if the same - * filter is not already defined in the filter profile. By enabling - * a domain and then still adding the filter to the filter profile + * filter is not already defined in the policy. By enabling + * a domain and then still adding the filter to the policy * the user can override this filter (can disable it, customize it, etc.) * @return A list of {@link AbstractFilter}. */ diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidFilterProfileException.java b/phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidPolicyException.java similarity index 79% rename from phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidFilterProfileException.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidPolicyException.java index 7f50bb3f..b581cec0 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidFilterProfileException.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/exceptions/InvalidPolicyException.java @@ -16,11 +16,11 @@ package ai.philterd.phileas.model.exceptions; /** - * Thrown when the requested filter profile does not exist. + * Thrown when the requested policy does not exist. */ -public class InvalidFilterProfileException extends RuntimeException { +public class InvalidPolicyException extends RuntimeException { - public InvalidFilterProfileException(String message) { + public InvalidPolicyException(String message) { super(message); } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/Filter.java b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/Filter.java index 68e64e43..d993de09 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/Filter.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/Filter.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.objects.*; import ai.philterd.phileas.model.profile.Crypto; import ai.philterd.phileas.model.profile.FPE; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.IgnoredPattern; import ai.philterd.phileas.model.profile.filters.Identifier; import ai.philterd.phileas.model.profile.filters.strategies.AbstractFilterStrategy; @@ -98,22 +98,22 @@ public abstract class Filter { /** * Filters the input text. - * @param filterProfile The {@link FilterProfile} to use. + * @param policy The {@link Policy} to use. * @param context The context. * @param documentId An ID uniquely identifying the document. * @param piece A numbered piece of the document. Pass 0 if only piece of document. * @param input The input text. * @return A {@link FilterResult} containing the identified {@link Span spans}. */ - public abstract FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String input) throws Exception; + public abstract FilterResult filter(Policy policy, String context, String documentId, int piece, String input) throws Exception; /** * Determines if the input text may contain sensitive information matching the filter type. - * @param filterProfile The {@link FilterProfile}. + * @param policy The {@link Policy}. * @param input The input text. * @return A count of possible occurrences of the filter type in the input text. */ - public abstract int getOccurrences(FilterProfile filterProfile, String input) throws Exception; + public abstract int getOccurrences(Policy policy, String input) throws Exception; /** * Creates a new filter. @@ -154,7 +154,7 @@ public Filter(FilterType filterType, FilterConfiguration filterConfiguration) { LOGGER.error("Unable to process file of ignored terms: {}", fileName, ex); } } else { - LOGGER.error("Ignore list file specified in filter profile does not exist: {}", fileName); + LOGGER.error("Ignore list file specified in policy does not exist: {}", fileName); } } } @@ -231,7 +231,7 @@ public String[] getWindow(String text, int characterStart, int characterEnd) { /** * Gets the string to be used as a replacement. - * @param filterProfile The name of the filter profile. + * @param policy The name of the policy. * @param context The context. * @param documentId The document ID. * @param token The token to replace. @@ -240,7 +240,7 @@ public String[] getWindow(String text, int characterStart, int characterEnd) { * @param classification The classification of the item. * @return The replacement string. */ - public Replacement getReplacement(String filterProfile, String context, String documentId, String token, String[] window, + public Replacement getReplacement(String policy, String context, String documentId, String token, String[] window, double confidence, String classification, FilterPattern filterPattern) throws Exception { if(strategies != null) { @@ -278,7 +278,7 @@ public Replacement getReplacement(String filterProfile, String context, String d if(strategy.isAlert()) { LOGGER.info("Generating alert for strategy ID {}", strategy.getId()); - alertService.generateAlert(filterProfile, strategy.getId(), documentId, context, filterType); + alertService.generateAlert(policy, strategy.getId(), documentId, context, filterType); } @@ -334,9 +334,9 @@ public boolean isIgnored(final String token) { } - public static List getIdentifierFilterStrategies(FilterProfile filterProfile, String name) { + public static List getIdentifierFilterStrategies(Policy policy, String name) { - final List identifiers = filterProfile.getIdentifiers().getIdentifiers(); + final List identifiers = policy.getIdentifiers().getIdentifiers(); final Identifier identifier = identifiers.stream(). filter(p -> p.getClassification().equalsIgnoreCase(name)). @@ -346,66 +346,66 @@ public static List getIdentifierFilterStrategi } - public static List getFilterStrategies(FilterProfile filterProfile, FilterType filterType, int index) { + public static List getFilterStrategies(Policy policy, FilterType filterType, int index) { LOGGER.debug("Getting filter strategies for filter type {}", filterType.getType()); if(filterType == FilterType.AGE) { - return filterProfile.getIdentifiers().getAge().getAgeFilterStrategies(); + return policy.getIdentifiers().getAge().getAgeFilterStrategies(); } else if(filterType == FilterType.BITCOIN_ADDRESS) { - return filterProfile.getIdentifiers().getBitcoinAddress().getBitcoinFilterStrategies(); + return policy.getIdentifiers().getBitcoinAddress().getBitcoinFilterStrategies(); } else if(filterType == FilterType.CREDIT_CARD) { - return filterProfile.getIdentifiers().getCreditCard().getCreditCardFilterStrategies(); + return policy.getIdentifiers().getCreditCard().getCreditCardFilterStrategies(); } else if(filterType == FilterType.CUSTOM_DICTIONARY) { - // There can be multiple custom dictionaries in the filter profile. + // There can be multiple custom dictionaries in the policy. // The index is used to determine which one is the appropriate one. - return filterProfile.getIdentifiers().getCustomDictionaries().get(index).getCustomDictionaryFilterStrategies(); + return policy.getIdentifiers().getCustomDictionaries().get(index).getCustomDictionaryFilterStrategies(); } else if(filterType == FilterType.DATE) { - return filterProfile.getIdentifiers().getDate().getDateFilterStrategies(); + return policy.getIdentifiers().getDate().getDateFilterStrategies(); } else if(filterType == FilterType.DRIVERS_LICENSE_NUMBER) { - return filterProfile.getIdentifiers().getDriversLicense().getDriversLicenseFilterStrategies(); + return policy.getIdentifiers().getDriversLicense().getDriversLicenseFilterStrategies(); } else if(filterType == FilterType.EMAIL_ADDRESS) { - return filterProfile.getIdentifiers().getEmailAddress().getEmailAddressFilterStrategies(); + return policy.getIdentifiers().getEmailAddress().getEmailAddressFilterStrategies(); } else if(filterType == FilterType.IBAN_CODE) { - return filterProfile.getIdentifiers().getIbanCode().getIbanCodeFilterStrategies(); + return policy.getIdentifiers().getIbanCode().getIbanCodeFilterStrategies(); } else if(filterType == FilterType.IP_ADDRESS) { - return filterProfile.getIdentifiers().getIpAddress().getIpAddressFilterStrategies(); + return policy.getIdentifiers().getIpAddress().getIpAddressFilterStrategies(); } else if(filterType == FilterType.PERSON) { - return filterProfile.getIdentifiers().getPerson().getNerStrategies(); + return policy.getIdentifiers().getPerson().getNerStrategies(); } else if(filterType == FilterType.PASSPORT_NUMBER) { - return filterProfile.getIdentifiers().getPassportNumber().getPassportNumberFilterStrategies(); + return policy.getIdentifiers().getPassportNumber().getPassportNumberFilterStrategies(); } else if(filterType == FilterType.PHONE_NUMBER) { - return filterProfile.getIdentifiers().getPhoneNumber().getPhoneNumberFilterStrategies(); + return policy.getIdentifiers().getPhoneNumber().getPhoneNumberFilterStrategies(); } else if(filterType == FilterType.PHONE_NUMBER_EXTENSION) { - return filterProfile.getIdentifiers().getPhoneNumberExtension().getPhoneNumberExtensionFilterStrategies(); + return policy.getIdentifiers().getPhoneNumberExtension().getPhoneNumberExtensionFilterStrategies(); } else if(filterType == FilterType.PHYSICIAN_NAME) { - return filterProfile.getIdentifiers().getPhysicianName().getPhysicianNameFilterStrategies(); + return policy.getIdentifiers().getPhysicianName().getPhysicianNameFilterStrategies(); } else if(filterType == FilterType.SSN) { - return filterProfile.getIdentifiers().getSsn().getSsnFilterStrategies(); + return policy.getIdentifiers().getSsn().getSsnFilterStrategies(); } else if(filterType == FilterType.STATE_ABBREVIATION) { - return filterProfile.getIdentifiers().getStateAbbreviation().getStateAbbreviationsFilterStrategies(); + return policy.getIdentifiers().getStateAbbreviation().getStateAbbreviationsFilterStrategies(); } else if(filterType == FilterType.STREET_ADDRESS) { - return filterProfile.getIdentifiers().getStreetAddress().getStreetAddressFilterStrategies(); + return policy.getIdentifiers().getStreetAddress().getStreetAddressFilterStrategies(); } else if(filterType == FilterType.URL) { - return filterProfile.getIdentifiers().getUrl().getUrlFilterStrategies(); + return policy.getIdentifiers().getUrl().getUrlFilterStrategies(); } else if(filterType == FilterType.VIN) { - return filterProfile.getIdentifiers().getVin().getVinFilterStrategies(); + return policy.getIdentifiers().getVin().getVinFilterStrategies(); } else if(filterType == FilterType.ZIP_CODE) { - return filterProfile.getIdentifiers().getZipCode().getZipCodeFilterStrategies(); + return policy.getIdentifiers().getZipCode().getZipCodeFilterStrategies(); } else if(filterType == FilterType.LOCATION_CITY) { - return filterProfile.getIdentifiers().getCity().getCityFilterStrategies(); + return policy.getIdentifiers().getCity().getCityFilterStrategies(); } else if(filterType == FilterType.LOCATION_COUNTY) { - return filterProfile.getIdentifiers().getCounty().getCountyFilterStrategies(); + return policy.getIdentifiers().getCounty().getCountyFilterStrategies(); } else if(filterType == FilterType.FIRST_NAME) { - return filterProfile.getIdentifiers().getFirstName().getFirstNameFilterStrategies(); + return policy.getIdentifiers().getFirstName().getFirstNameFilterStrategies(); } else if(filterType == FilterType.HOSPITAL_ABBREVIATION) { - return filterProfile.getIdentifiers().getHospitalAbbreviation().getHospitalAbbreviationFilterStrategies(); + return policy.getIdentifiers().getHospitalAbbreviation().getHospitalAbbreviationFilterStrategies(); } else if(filterType == FilterType.HOSPITAL) { - return filterProfile.getIdentifiers().getHospital().getHospitalFilterStrategies(); + return policy.getIdentifiers().getHospital().getHospitalFilterStrategies(); } else if(filterType == FilterType.LOCATION_STATE) { - return filterProfile.getIdentifiers().getState().getStateFilterStrategies(); + return policy.getIdentifiers().getState().getStateFilterStrategies(); } else if(filterType == FilterType.SURNAME) { - return filterProfile.getIdentifiers().getSurname().getSurnameFilterStrategies(); + return policy.getIdentifiers().getSurname().getSurnameFilterStrategies(); } // Should never happen. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/RulesFilter.java b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/RulesFilter.java index a2a59647..2713d798 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/RulesFilter.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/RulesFilter.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.filter.Filter; import ai.philterd.phileas.model.filter.FilterConfiguration; import ai.philterd.phileas.model.objects.*; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import java.util.*; import java.util.regex.Matcher; @@ -55,21 +55,21 @@ public List postFilter(List spans) { /** * Find {@link Span spans} matching the {@link Pattern}. - * @param filterProfile The {@link FilterProfile} to use. + * @param policy The {@link Policy} to use. * @param analyzer A filter {@link Analyzer}. * @param input The text input. * @param context The context. * @param documentId The document ID. * @return A list of matching {@link Span spans}. */ - protected List findSpans(FilterProfile filterProfile, Analyzer analyzer, String input, String context, + protected List findSpans(Policy policy, Analyzer analyzer, String input, String context, String documentId) throws Exception { - return findSpans(filterProfile, analyzer, input, context, documentId, Collections.emptyMap()); + return findSpans(policy, analyzer, input, context, documentId, Collections.emptyMap()); } /** * Find {@link Span spans} matching the {@link Pattern}. - * @param filterProfile The {@link FilterProfile} to use. + * @param policy The {@link Policy} to use. * @param analyzer A filter {@link Analyzer}. * @param input The text input. * @param context The context. @@ -77,13 +77,13 @@ protected List findSpans(FilterProfile filterProfile, Analyzer analyzer, S * @param restrictions Restrictions placed on what is a span for the filter. * @return A list of matching {@link Span spans}. */ - protected List findSpans(FilterProfile filterProfile, Analyzer analyzer, String input, String context, + protected List findSpans(Policy policy, Analyzer analyzer, String input, String context, String documentId, Map> restrictions) throws Exception { final List spans = new LinkedList<>(); - // Is this filter enabled for this filter profile? If not just return empty list. - if(filterProfile.getIdentifiers().hasFilter(filterType)) { + // Is this filter enabled for this policy? If not just return empty list. + if(policy.getIdentifiers().hasFilter(filterType)) { for(final FilterPattern filterPattern : analyzer.getFilterPatterns()) { @@ -112,7 +112,7 @@ protected List findSpans(FilterProfile filterProfile, Analyzer analyzer, S final String[] window = getWindow(input, characterStart, characterEnd); // Get the span's replacement. - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, token, + final Replacement replacement = getReplacement(policy.getName(), context, documentId, token, window, initialConfidence, filterPattern.getClassification(), filterPattern); final Span span = Span.make(characterStart, characterEnd, getFilterType(), context, documentId, @@ -155,14 +155,14 @@ protected List findSpans(FilterProfile filterProfile, Analyzer analyzer, S /** * Gets the count of occurrences. - * @param filterProfile The {@link FilterProfile} to use. + * @param policy The {@link Policy} to use. * @param input The input text. * @return A count of occurrences in the text. */ @Override - public int getOccurrences(FilterProfile filterProfile, String input) throws Exception { + public int getOccurrences(Policy policy, String input) throws Exception { - return filter(filterProfile, "none", "none", 0, input).getSpans().size(); + return filter(policy, "none", "none", 0, input).getSpans().size(); } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/BloomFilterDictionaryFilter.java b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/BloomFilterDictionaryFilter.java index c375f154..fbbc0f1f 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/BloomFilterDictionaryFilter.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/BloomFilterDictionaryFilter.java @@ -19,11 +19,10 @@ import com.google.common.hash.Funnels; import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.filter.FilterConfiguration; -import ai.philterd.phileas.model.objects.DocumentAnalysis; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.shingle.ShingleFilter; @@ -84,7 +83,7 @@ public BloomFilterDictionaryFilter(FilterType filterType, } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String text) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String text) throws Exception { final List spans = new LinkedList<>(); @@ -117,7 +116,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d // Get the original token to get the right casing. final String originalToken = text.substring(characterStart, characterEnd); - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, + final Replacement replacement = getReplacement(policy.getName(), context, documentId, originalToken, window, confidence, classification, null); spans.add(Span.make(characterStart, characterEnd, getFilterType(), context, documentId, diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/LuceneDictionaryFilter.java b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/LuceneDictionaryFilter.java index afa8f53a..c498df55 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/LuceneDictionaryFilter.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/filter/rules/dictionary/LuceneDictionaryFilter.java @@ -18,11 +18,10 @@ import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.enums.SensitivityLevel; import ai.philterd.phileas.model.filter.FilterConfiguration; -import ai.philterd.phileas.model.objects.DocumentAnalysis; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Replacement; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.text.similarity.LevenshteinDistance; @@ -65,7 +64,7 @@ public class LuceneDictionaryFilter extends DictionaryFilter implements Serializ private LevenshteinDistance distanceFunction; private SensitivityLevel sensitivityLevel; private boolean capitalized; - private int filterProfileIndex = 0; + private int policyIndex = 0; /** * Creates a new Lucene dictionary filter. @@ -103,7 +102,7 @@ public LuceneDictionaryFilter(FilterType filterType, * @param terms * @param capitalized * @param type - * @param filterProfileIndex + * @param policyIndex * @throws IOException Thrown if the index cannot be opened or accessed. */ public LuceneDictionaryFilter(FilterType filterType, @@ -112,7 +111,7 @@ public LuceneDictionaryFilter(FilterType filterType, Set terms, boolean capitalized, String type, - int filterProfileIndex) throws IOException { + int policyIndex) throws IOException { super(filterType, filterConfiguration); @@ -121,7 +120,7 @@ public LuceneDictionaryFilter(FilterType filterType, this.distanceFunction = new LevenshteinDistance(); this.sensitivityLevel = sensitivityLevel; this.capitalized = capitalized; - this.filterProfileIndex = filterProfileIndex; + this.policyIndex = policyIndex; // Find the max n-gram size. It is equal to the maximum // number of spaces in any single dictionary entry. @@ -165,11 +164,11 @@ public void close() throws IOException { } @Override - public FilterResult filter(FilterProfile filterProfile, String context, String documentId, int piece, String text) throws Exception { + public FilterResult filter(Policy policy, String context, String documentId, int piece, String text) throws Exception { final List spans = new LinkedList<>(); - if(filterProfile.getIdentifiers().hasFilter(filterType)) { + if(policy.getIdentifiers().hasFilter(filterType)) { try(final Analyzer analyzer = new StandardAnalyzer()) { @@ -274,7 +273,7 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d final String classification = ""; // Get the replacement token or the original token if no filter strategy conditions are met. - final Replacement replacement = getReplacement(filterProfile.getName(), context, documentId, token, window, confidence, classification, null); + final Replacement replacement = getReplacement(policy.getName(), context, documentId, token, window, confidence, classification, null); // Add the span to the list. spans.add(Span.make(characterStart, characterEnd, getFilterType(), context, documentId, confidence, token, replacement.getReplacement(), replacement.getSalt(), ignored, window)); @@ -309,9 +308,9 @@ public FilterResult filter(FilterProfile filterProfile, String context, String d } @Override - public int getOccurrences(FilterProfile filterProfile, String input) throws Exception { + public int getOccurrences(Policy policy, String input) throws Exception { - return filter(filterProfile, "none", "none", 0, input).getSpans().size(); + return filter(policy, "none", "none", 0, input).getSpans().size(); } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/objects/Alert.java b/phileas-model/src/main/java/ai/philterd/phileas/model/objects/Alert.java index 7506df5c..2dfc2007 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/objects/Alert.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/objects/Alert.java @@ -30,7 +30,7 @@ public class Alert implements Serializable { private final transient DateFormat utcDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); private String id; - private String filterProfile; + private String policy; private String strategyId; private String context; private String documentId; @@ -41,10 +41,10 @@ public Alert() { } - public Alert(String filterProfile, String strategyId, String context, String documentId, String filterType) { + public Alert(String policy, String strategyId, String context, String documentId, String filterType) { this.id = UUID.randomUUID().toString(); - this.filterProfile = filterProfile; + this.policy = policy; this.strategyId = strategyId; this.context = context; this.documentId = documentId; @@ -70,7 +70,7 @@ public boolean equals(Object o) { public int hashCode() { return new HashCodeBuilder(17, 37). append(id). - append(filterProfile). + append(policy). append(strategyId). append(context). append(documentId). @@ -127,12 +127,12 @@ public void setId(String id) { this.id = id; } - public String getFilterProfile() { - return filterProfile; + public String getPolicy() { + return policy; } - public void setFilterProfile(String filterProfile) { - this.filterProfile = filterProfile; + public void setPolicy(String policy) { + this.policy = policy; } } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetFilterProfileResult.java b/phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetPolicyResult.java similarity index 74% rename from phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetFilterProfileResult.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetPolicyResult.java index 66534ac1..5d37f9cd 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetFilterProfileResult.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/objects/GetPolicyResult.java @@ -15,20 +15,20 @@ */ package ai.philterd.phileas.model.objects; -public class GetFilterProfileResult { +public class GetPolicyResult { - private String filterProfileJson; + private String policyJson; private boolean requiresReload; - public GetFilterProfileResult(String filterProfileJson, boolean requiresReload) { + public GetPolicyResult(String policyJson, boolean requiresReload) { - this.filterProfileJson = filterProfileJson; + this.policyJson = policyJson; this.requiresReload = requiresReload; } - public String getFilterProfileJson() { - return filterProfileJson; + public String getPolicyJson() { + return policyJson; } public boolean isRequiresReload() { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Identifiers.java b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Identifiers.java index 811567c9..d35f4b40 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Identifiers.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Identifiers.java @@ -174,9 +174,9 @@ public Identifiers() { } /** - * Determines if a filter exists for a filter profile. + * Determines if a filter exists for a policy. * @param filterType The {@link FilterType}. - * @return true if the filter profile contains a strategy for the corresponding + * @return true if the policy contains a strategy for the corresponding * {@link FilterType}; otherwise false. */ public boolean hasFilter(FilterType filterType) { diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/FilterProfile.java b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Policy.java similarity index 99% rename from phileas-model/src/main/java/ai/philterd/phileas/model/profile/FilterProfile.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/profile/Policy.java index ca992016..80f40c58 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/FilterProfile.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/Policy.java @@ -23,7 +23,7 @@ import java.util.Collections; import java.util.List; -public class FilterProfile { +public class Policy { @SerializedName("name") @Expose diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/filters/Identifier.java b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/filters/Identifier.java index ec0fc9ec..8372ec7e 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/profile/filters/Identifier.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/profile/filters/Identifier.java @@ -24,7 +24,7 @@ public class Identifier extends AbstractFilter { /** - * The default regex pattern to use if none is provided in the filter profile. + * The default regex pattern to use if none is provided in the policy. */ public static final String DEFAULT_IDENTIFIER_REGEX = "\\b[A-Z0-9_-]{6,}\\b"; diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractFilterProfileService.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractPolicyService.java similarity index 93% rename from phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractFilterProfileService.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractPolicyService.java index 76933ce9..c6d11241 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractFilterProfileService.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/AbstractPolicyService.java @@ -15,6 +15,6 @@ */ package ai.philterd.phileas.model.services; -public class AbstractFilterProfileService { +public class AbstractPolicyService { } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/AlertService.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/AlertService.java index cf2a12e9..dc4eeed3 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/AlertService.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/AlertService.java @@ -25,13 +25,13 @@ public interface AlertService { /** * Generate an alert. * - * @param filterProfile The name of the filter profile. + * @param policy The name of the policy. * @param strategyId The ID of the filter strategy that caused the alert. * @param context The context. * @param documentId The document ID. * @param filterType The {@link FilterType}. */ - void generateAlert(String filterProfile, String strategyId, String context, String documentId, FilterType filterType); + void generateAlert(String policy, String strategyId, String context, String documentId, FilterType filterType); /** * Gets all of the alerts. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/DocumentProcessor.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/DocumentProcessor.java index 84817c4f..be66dc54 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/DocumentProcessor.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/DocumentProcessor.java @@ -16,7 +16,7 @@ package ai.philterd.phileas.model.services; import ai.philterd.phileas.model.filter.Filter; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.responses.FilterResponse; import java.util.List; @@ -26,6 +26,6 @@ */ public interface DocumentProcessor { - FilterResponse process(FilterProfile filterProfile, List filters, List postFilters, String context, String documentId, int piece, String input) throws Exception; + FilterResponse process(Policy policy, List filters, List postFilters, String context, String documentId, int piece, String input) throws Exception; } diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterService.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterService.java index 365611af..3e34c16a 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterService.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterService.java @@ -28,7 +28,7 @@ public interface FilterService { /** * Filter text from a string. - * @param filterProfileNames The list of the filter profiles to use. + * @param policyNames The list of the policies to use. * @param context The context. * @param documentId A document ID. Provide null for a document ID to be generated. * @param input The input text. @@ -36,11 +36,11 @@ public interface FilterService { * @return A {@link FilterResponse}. * @throws Exception Thrown if the text cannot be filtered. */ - FilterResponse filter(List filterProfileNames, String context, String documentId, String input, MimeType mimeType) throws Exception; + FilterResponse filter(List policyNames, String context, String documentId, String input, MimeType mimeType) throws Exception; /** * Filter text from a binary document. - * @param filterProfileNames The list of the filter profiles to use. + * @param policyNames The list of the policies to use. * @param context The context. * @param documentId A document ID. Provide null for a document ID to be generated. * @param input The input document as a byte array. @@ -49,13 +49,13 @@ public interface FilterService { * @return A {@link BinaryDocumentFilterResponse}. * @throws Exception Thrown if the text cannot be filtered. */ - BinaryDocumentFilterResponse filter(List filterProfileNames, String context, String documentId, byte[] input, MimeType mimeType, MimeType outputMimeType) throws Exception; + BinaryDocumentFilterResponse filter(List policyNames, String context, String documentId, byte[] input, MimeType mimeType, MimeType outputMimeType) throws Exception; /** - * Gets the filter profile service being used. - * @return A {@link FilterProfileService}. + * Gets the policy service being used. + * @return A {@link PolicyService}. */ - FilterProfileService getFilterProfileService(); + PolicyService getPolicyService(); /** * Returns the alert service being used. diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileCacheService.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyCacheService.java similarity index 60% rename from phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileCacheService.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyCacheService.java index 1e375edd..b609cf5d 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileCacheService.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyCacheService.java @@ -19,45 +19,45 @@ import java.util.List; import java.util.Map; -public interface FilterProfileCacheService { +public interface PolicyCacheService { /** - * Gets the names of all filter profiles. + * Gets the names of all policies. * @return * @throws IOException */ List get() throws IOException; /** - * Gets the content of a filter profile. - * @param filterProfileName + * Gets the content of a policy. + * @param policyName * @return * @throws IOException */ - String get(String filterProfileName) throws IOException; + String get(String policyName) throws IOException; /** - * Get the names and content of all filter profiles. + * Get the names and content of all policies. * @return * @throws IOException */ Map getAll() throws IOException; /** - * Inserts a new filter profile into the cache. - * @param filterProfileName The name of the filter profile. - * @param filterProfile The content of the filter profile. + * Inserts a new policy into the cache. + * @param policyName The name of the policy. + * @param policy The content of the policy. */ - void insert(String filterProfileName, String filterProfile); + void insert(String policyName, String policy); /** - * Removes a filter profile from the cache. - * @param filterProfileName The name of the filter profile. + * Removes a policy from the cache. + * @param policyName The name of the policy. */ - void remove(String filterProfileName); + void remove(String policyName); /** - * Clears all filter profiles from the cache. + * Clears all policies from the cache. * @throws IOException */ void clear() throws IOException; diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileService.java b/phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyService.java similarity index 59% rename from phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileService.java rename to phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyService.java index 5719d73f..fea25e04 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/services/FilterProfileService.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/services/PolicyService.java @@ -19,43 +19,43 @@ import java.util.List; import java.util.Map; -public interface FilterProfileService { +public interface PolicyService { /** - * Gets the names of all filter profiles from + * Gets the names of all policies from * the backend store directly by-passing the cache. - * @return A list of filter profile names. + * @return A list of policy names. * @throws IOException */ List get() throws IOException; /** - * Gets the content of a filter profile. - * @param filterProfileName - * @return The filter profile. + * Gets the content of a policy. + * @param policyName + * @return The policy. * @throws IOException */ - String get(String filterProfileName) throws IOException; + String get(String policyName) throws IOException; /** - * Get the names and content of all filter profiles. - * @return A map of filter profile names to filter profile content. + * Get the names and content of all policies. + * @return A map of policy names to policy content. * @throws IOException */ Map getAll() throws IOException; /** - * Saves a filter profile. - * @param filterProfileJson The content of the filter profile as JSON. + * Saves a policy. + * @param policyJson The content of the policy as JSON. * @throws IOException */ - void save(String filterProfileJson) throws IOException; + void save(String policyJson) throws IOException; /** - * Deletes a filter profile. - * @param filterProfileName The name of the filter profile to delete. + * Deletes a policy. + * @param policyName The name of the policy to delete. * @throws IOException */ - void delete(String filterProfileName) throws IOException; + void delete(String policyName) throws IOException; } diff --git a/phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/FilterProfileTest.java b/phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/PolicyTest.java similarity index 93% rename from phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/FilterProfileTest.java rename to phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/PolicyTest.java index 8b133467..ee1851c0 100644 --- a/phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/FilterProfileTest.java +++ b/phileas-model/src/test/java/ai/philterd/test/phileas/model/profile/PolicyTest.java @@ -19,7 +19,7 @@ import com.google.gson.GsonBuilder; import ai.philterd.phileas.model.enums.FilterType; import ai.philterd.phileas.model.enums.SensitivityLevel; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Identifiers; import ai.philterd.phileas.model.profile.Ignored; import ai.philterd.phileas.model.profile.filters.*; @@ -34,7 +34,7 @@ import java.io.IOException; import java.util.Arrays; -public class FilterProfileTest { +public class PolicyTest { @Test public void serialize() throws IOException { @@ -140,13 +140,13 @@ public void serialize() throws IOException { ignored.setName("ignored-terms"); ignored.setTerms(Arrays.asList("term1", "term2")); - FilterProfile filterProfile = new FilterProfile(); - filterProfile.setName("default"); - filterProfile.setIdentifiers(identifiers); - filterProfile.setIgnored(Arrays.asList(ignored)); + Policy policy = new Policy(); + policy.setName("default"); + policy.setIdentifiers(identifiers); + policy.setIgnored(Arrays.asList(ignored)); Gson gson = new GsonBuilder().serializeNulls().setPrettyPrinting().create(); - String json = gson.toJson(filterProfile); + String json = gson.toJson(policy); System.out.println(json); @@ -289,11 +289,11 @@ public void deserialize1() { "}\n"; Gson gson = new Gson(); - FilterProfile filterProfile = gson.fromJson(json, FilterProfile.class); + Policy policy = gson.fromJson(json, Policy.class); - Assertions.assertTrue(CollectionUtils.isNotEmpty(filterProfile.getIdentifiers().getCustomDictionaries())); - Assertions.assertTrue(CollectionUtils.isNotEmpty(filterProfile.getIgnored())); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); + Assertions.assertTrue(CollectionUtils.isNotEmpty(policy.getIdentifiers().getCustomDictionaries())); + Assertions.assertTrue(CollectionUtils.isNotEmpty(policy.getIgnored())); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); } @@ -403,11 +403,11 @@ public void deserialize2() { "}\n"; Gson gson = new Gson(); - FilterProfile filterProfile = gson.fromJson(json, FilterProfile.class); + Policy policy = gson.fromJson(json, Policy.class); - Assertions.assertFalse(CollectionUtils.isNotEmpty(filterProfile.getIdentifiers().getCustomDictionaries())); - Assertions.assertFalse(CollectionUtils.isNotEmpty(filterProfile.getIgnored())); - Assertions.assertFalse(filterProfile.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); + Assertions.assertFalse(CollectionUtils.isNotEmpty(policy.getIdentifiers().getCustomDictionaries())); + Assertions.assertFalse(CollectionUtils.isNotEmpty(policy.getIgnored())); + Assertions.assertFalse(policy.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); } @@ -420,7 +420,7 @@ public void deserializeEmpty() { "}"; Gson gson = new Gson(); - FilterProfile filterProfile = gson.fromJson(json, FilterProfile.class); + Policy policy = gson.fromJson(json, Policy.class); } diff --git a/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/main/java/ai/philterd/phileas/processors/structured/fhir/FhirDocumentProcessor.java b/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/main/java/ai/philterd/phileas/processors/structured/fhir/FhirDocumentProcessor.java index 3b6ef7c2..d7867efb 100644 --- a/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/main/java/ai/philterd/phileas/processors/structured/fhir/FhirDocumentProcessor.java +++ b/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/main/java/ai/philterd/phileas/processors/structured/fhir/FhirDocumentProcessor.java @@ -19,7 +19,7 @@ import ca.uhn.fhir.parser.IParser; import ai.philterd.phileas.model.filter.Filter; import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.fhir4.FhirItem; import ai.philterd.phileas.model.profile.fhir4.FhirR4; import ai.philterd.phileas.model.responses.FilterResponse; @@ -56,16 +56,16 @@ public FhirDocumentProcessor(MetricsService metricsService, SpanDisambiguationSe } @Override - public FilterResponse process(FilterProfile filterProfile, List filters, List postFilters, + public FilterResponse process(Policy policy, List filters, List postFilters, String context, String documentId, String json) throws Exception { // TODO: I'm getting FhirR4 here but that version is really unknown to the API. // All we know is that it is an application/fhir+json document. // Should the version be passed in as an API header or something? - LOGGER.debug("Doing FHIRv4 processing with filter profile [{}]", filterProfile.getName()); + LOGGER.debug("Doing FHIRv4 processing with policy [{}]", policy.getName()); - final FhirR4 fhirR4 = filterProfile.getStructured().getFhirR4(); + final FhirR4 fhirR4 = policy.getStructured().getFhirR4(); final List fhirItems = fhirR4.getFhirItems(); final FhirContext ctx = FhirContext.forR4(); @@ -74,7 +74,7 @@ public FilterResponse process(FilterProfile filterProfile, List filters, final Patient patient = parser.parseResource(Patient.class, json); // Used for value encryption. May not be needed. - final Crypto crypto = filterProfile.getCrypto(); + final Crypto crypto = policy.getCrypto(); // Make the changes in the Patient document. diff --git a/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/test/java/ai/philterd/test/phileas/processors/structured/fhir/FhirDocumentProcessorTest.java b/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/test/java/ai/philterd/test/phileas/processors/structured/fhir/FhirDocumentProcessorTest.java index 4270e804..449e162e 100644 --- a/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/test/java/ai/philterd/test/phileas/processors/structured/fhir/FhirDocumentProcessorTest.java +++ b/phileas-processors/phileas-processors-structured/phileas-processors-structured-fhir/src/test/java/ai/philterd/test/phileas/processors/structured/fhir/FhirDocumentProcessorTest.java @@ -23,7 +23,7 @@ import com.google.gson.JsonParser; import ai.philterd.phileas.processors.structured.fhir.FhirDocumentProcessor; import ai.philterd.phileas.model.profile.Crypto; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.Structured; import ai.philterd.phileas.model.profile.fhir4.FhirItem; import ai.philterd.phileas.model.profile.fhir4.FhirR4; @@ -54,9 +54,9 @@ public void test1() throws Exception { final SpanDisambiguationService spanDisambiguationService = Mockito.mock(SpanDisambiguationService.class); final DocumentProcessor documentProcessor = new FhirDocumentProcessor(metricsService, spanDisambiguationService); - // FilterProfile filterProfile, String context, String documentId, String json - final FilterProfile filterProfile = new FilterProfile(); - filterProfile.setCrypto(new Crypto("key", "iv")); + // Policy policy, String context, String documentId, String json + final Policy policy = new Policy(); + policy.setCrypto(new Crypto("key", "iv")); final Structured structured = new Structured(); @@ -65,14 +65,14 @@ public void test1() throws Exception { //fhirR4.setFhirItems(Arrays.asList(new FhirItem("patient.address.city", "DELETE"))); structured.setFhirR4(fhirR4); - filterProfile.setStructured(structured); + policy.setStructured(structured); - prettyPrintJson(filterProfile); + prettyPrintJson(policy); final String json = IOUtils.toString(this.getClass().getResourceAsStream("/fhir4/bundle-example.json"), Charset.defaultCharset()); // TODO: Set filters instead of empty list. - final FilterResponse filterResponse = documentProcessor.process(filterProfile, Collections.emptyList(), Collections.emptyList(), "context", "documentid", json); + final FilterResponse filterResponse = documentProcessor.process(policy, Collections.emptyList(), Collections.emptyList(), "context", "documentid", json); prettyPrintJson(filterResponse.getFilteredText()); diff --git a/phileas-processors/phileas-processors-unstructured/src/main/java/ai/philterd/phileas/processors/unstructured/UnstructuredDocumentProcessor.java b/phileas-processors/phileas-processors-unstructured/src/main/java/ai/philterd/phileas/processors/unstructured/UnstructuredDocumentProcessor.java index cb7c7160..49628ca5 100644 --- a/phileas-processors/phileas-processors-unstructured/src/main/java/ai/philterd/phileas/processors/unstructured/UnstructuredDocumentProcessor.java +++ b/phileas-processors/phileas-processors-unstructured/src/main/java/ai/philterd/phileas/processors/unstructured/UnstructuredDocumentProcessor.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.objects.Explanation; import ai.philterd.phileas.model.objects.FilterResult; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.responses.FilterResponse; import ai.philterd.phileas.model.services.DocumentProcessor; import ai.philterd.phileas.model.services.MetricsService; @@ -48,7 +48,7 @@ public UnstructuredDocumentProcessor(MetricsService metricsService, SpanDisambig } @Override - public FilterResponse process(FilterProfile filterProfile, List filters, List postFilters, + public FilterResponse process(Policy policy, List filters, List postFilters, String context, String documentId, int piece, String input) throws Exception { // The list that will contain the spans containing PHI/PII. @@ -58,7 +58,7 @@ public FilterResponse process(FilterProfile filterProfile, List filters, for(final Filter filter : filters) { final long startTimeMs = System.currentTimeMillis(); - final FilterResult filterResult = filter.filter(filterProfile, context, documentId, piece, input); + final FilterResult filterResult = filter.filter(policy, context, documentId, piece, input); final long elapsedTimeMs = System.currentTimeMillis() - startTimeMs; metricsService.logFilterTime(filter.getFilterType(), elapsedTimeMs); diff --git a/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/LocalAlertService.java b/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/LocalAlertService.java index 9811dfa4..d9b6b4d5 100644 --- a/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/LocalAlertService.java +++ b/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/LocalAlertService.java @@ -39,9 +39,9 @@ public LocalAlertService() { } @Override - public void generateAlert(String filterProfile, String strategyId, String context, String documentId, FilterType filterType) { + public void generateAlert(String policy, String strategyId, String context, String documentId, FilterType filterType) { - final Alert alert = new Alert(filterProfile, strategyId, context, documentId, filterType.getType()); + final Alert alert = new Alert(policy, strategyId, context, documentId, filterType.getType()); alerts.add(alert); diff --git a/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/RedisAlertService.java b/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/RedisAlertService.java index bb09286d..2a16a9ef 100644 --- a/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/RedisAlertService.java +++ b/phileas-services/phileas-services-alerts/src/main/java/ai/philterd/phileas/services/alerts/RedisAlertService.java @@ -40,9 +40,9 @@ public RedisAlertService(PhileasConfiguration phileasConfiguration) throws IOExc } @Override - public void generateAlert(String filterProfile, String strategyId, String context, String documentId, FilterType filterType) { + public void generateAlert(String policy, String strategyId, String context, String documentId, FilterType filterType) { - final Alert alert = new Alert(filterProfile, strategyId, context, documentId, filterType.getType()); + final Alert alert = new Alert(policy, strategyId, context, documentId, filterType.getType()); redisson.getList(CACHE_LIST_NAME).add(alert); diff --git a/phileas-services/phileas-services-pdf/src/main/java/ai/philterd/services/pdf/PdfRedacter.java b/phileas-services/phileas-services-pdf/src/main/java/ai/philterd/services/pdf/PdfRedacter.java index 04b16113..e3c746ed 100644 --- a/phileas-services/phileas-services-pdf/src/main/java/ai/philterd/services/pdf/PdfRedacter.java +++ b/phileas-services/phileas-services-pdf/src/main/java/ai/philterd/services/pdf/PdfRedacter.java @@ -18,7 +18,7 @@ import ai.philterd.phileas.model.enums.MimeType; import ai.philterd.phileas.model.objects.PdfRedactionOptions; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.graphical.BoundingBox; import ai.philterd.phileas.model.services.Redacter; import ai.philterd.services.pdf.model.RedactedRectangle; @@ -31,7 +31,6 @@ import org.apache.pdfbox.pdmodel.graphics.color.PDColor; import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB; import org.apache.pdfbox.pdmodel.graphics.image.JPEGFactory; -import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory; import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; import org.apache.pdfbox.pdmodel.graphics.state.RenderingMode; import org.apache.pdfbox.rendering.ImageType; @@ -58,7 +57,7 @@ public class PdfRedacter extends PDFTextStripper implements Redacter { private Map> rectangles = new HashMap<>(); - private FilterProfile filterProfile; + private Policy policy; private final Set spans; private final PdfRedactionOptions pdfRedactionOptions; private final List boundingBoxes; @@ -71,11 +70,11 @@ public class PdfRedacter extends PDFTextStripper implements Redacter { COLORS.put("yellow", new PDColor(new float[]{1, 1, 100 / 255F}, PDDeviceRGB.INSTANCE)); } - public PdfRedacter(FilterProfile filterProfile, + public PdfRedacter(Policy policy, Set spans, PdfRedactionOptions pdfRedactionOptions, List boundingBoxes) throws IOException { - this.filterProfile = filterProfile; + this.policy = policy; this.spans = spans; this.pdfRedactionOptions = pdfRedactionOptions; this.boundingBoxes = boundingBoxes; @@ -223,7 +222,7 @@ protected void endDocument(PDDocument doc) throws IOException { } // Get the color based on the filter. - final PDColor pdColor = COLORS.getOrDefault(filterProfile.getConfig().getPdf().getRedactionColor(), COLORS.get("black")); + final PDColor pdColor = COLORS.getOrDefault(policy.getConfig().getPdf().getRedactionColor(), COLORS.get("black")); contentStream.setNonStrokingColor(pdColor); contentStream.setRenderingMode(RenderingMode.FILL); contentStream.fill(); diff --git a/phileas-services/phileas-services-pdf/src/test/java/ai/philterd/PdfRedacterTest.java b/phileas-services/phileas-services-pdf/src/test/java/ai/philterd/PdfRedacterTest.java index 4cbd3ab5..ee429a22 100644 --- a/phileas-services/phileas-services-pdf/src/test/java/ai/philterd/PdfRedacterTest.java +++ b/phileas-services/phileas-services-pdf/src/test/java/ai/philterd/PdfRedacterTest.java @@ -19,7 +19,7 @@ import ai.philterd.phileas.model.enums.MimeType; import ai.philterd.phileas.model.objects.PdfRedactionOptions; import ai.philterd.phileas.model.objects.Span; -import ai.philterd.phileas.model.profile.FilterProfile; +import ai.philterd.phileas.model.profile.Policy; import ai.philterd.phileas.model.profile.graphical.BoundingBox; import ai.philterd.phileas.model.services.Redacter; import ai.philterd.services.pdf.PdfRedacter; @@ -54,7 +54,7 @@ public void testPDF1() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); pdfRedactionOptions.setDpi(150); @@ -62,7 +62,7 @@ public void testPDF1() throws IOException { pdfRedactionOptions.setCompressionQuality(1.0f); final List boundingBoxes = Collections.emptyList(); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.APPLICATION_PDF); @@ -86,11 +86,11 @@ public void testPDF2() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final List boundingBoxes = Collections.emptyList(); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.APPLICATION_PDF); @@ -115,11 +115,11 @@ public void testJpeg1() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final List boundingBoxes = Collections.emptyList(); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.IMAGE_JPEG); final File outputFile = File.createTempFile("output", ".zip"); @@ -142,11 +142,11 @@ public void testJpeg2() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final List boundingBoxes = Collections.emptyList(); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.IMAGE_JPEG); final File outputFile = File.createTempFile("output", ".zip"); @@ -166,7 +166,7 @@ public void textPdfBoundingBox1() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final BoundingBox boundingBox1 = new BoundingBox(); @@ -185,7 +185,7 @@ public void textPdfBoundingBox1() throws IOException { final List boundingBoxes = Arrays.asList(boundingBox1, boundingBox2); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, Collections.emptySet(), pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, Collections.emptySet(), pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.APPLICATION_PDF); final File outputFile = File.createTempFile("output", ".pdf"); @@ -203,7 +203,7 @@ public void textPdfBoundingBox2() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final BoundingBox boundingBox1 = new BoundingBox(); @@ -226,7 +226,7 @@ public void textPdfBoundingBox2() throws IOException { final List boundingBoxes = Arrays.asList(boundingBox1, boundingBox2); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, Collections.emptySet(), pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, Collections.emptySet(), pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.APPLICATION_PDF); final File outputFile = File.createTempFile("output", ".pdf"); @@ -248,7 +248,7 @@ public void testPdfSpansAndBoundingBoxes() throws IOException { final InputStream is = getClass().getClassLoader().getResourceAsStream(filename); final byte[] document = IOUtils.toByteArray(is); - final FilterProfile filterProfile = new FilterProfile(); + final Policy policy = new Policy(); final PdfRedactionOptions pdfRedactionOptions = new PdfRedactionOptions(); final BoundingBox boundingBox1 = new BoundingBox(); @@ -267,7 +267,7 @@ public void testPdfSpansAndBoundingBoxes() throws IOException { final List boundingBoxes = Arrays.asList(boundingBox1, boundingBox2); - final Redacter pdfRedacter = new PdfRedacter(filterProfile, spans, pdfRedactionOptions, boundingBoxes); + final Redacter pdfRedacter = new PdfRedacter(policy, spans, pdfRedactionOptions, boundingBoxes); final byte[] redacted = pdfRedacter.process(document, MimeType.APPLICATION_PDF); final File outputFile = File.createTempFile("output", ".pdf"); diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalFilterProfileService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalFilterProfileService.java deleted file mode 100644 index 1329d1be..00000000 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalFilterProfileService.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2023 Philterd, LLC @ https://www.philterd.ai - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.philterd.phileas.services.profiles; - -import ai.philterd.phileas.configuration.PhileasConfiguration; -import ai.philterd.phileas.model.exceptions.api.BadRequestException; -import ai.philterd.phileas.model.services.AbstractFilterProfileService; -import ai.philterd.phileas.model.services.FilterProfileCacheService; -import ai.philterd.phileas.model.services.FilterProfileService; -import ai.philterd.phileas.services.profiles.cache.InMemoryFilterProfileCacheService; -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.*; - -public class LocalFilterProfileService extends AbstractFilterProfileService implements FilterProfileService { - - private static final Logger LOGGER = LogManager.getLogger(LocalFilterProfileService.class); - - private static final String JSON_EXTENSION = ".json"; - - private String filterProfilesDirectory; - private FilterProfileCacheService filterProfileCacheService; - - public LocalFilterProfileService(PhileasConfiguration phileasConfiguration) { - - this.filterProfilesDirectory = phileasConfiguration.filterProfilesDirectory(); - LOGGER.info("Looking for filter profiles in {}", filterProfilesDirectory); - - // Always use an in-memory cache when using a local filter profile service. - this.filterProfileCacheService = new InMemoryFilterProfileCacheService(); - - } - - @Override - public List get() throws IOException { - - // This function never uses a cache. - - final List names = new LinkedList<>(); - - // Read the filter profiles from the file system. - final Collection files = FileUtils.listFiles(new File(filterProfilesDirectory), new String[]{"json"}, false); - - for(final File file : files) { - - final String json = FileUtils.readFileToString(file, Charset.defaultCharset()); - - final JSONObject object = new JSONObject(json); - final String name = object.getString("name"); - - names.add(name); - - } - - return names; - - } - - @Override - public String get(String filterProfileName) throws IOException { - - String filterProfileJson = filterProfileCacheService.get(filterProfileName); - - if(filterProfileJson == null) { - - // The filter profile wasn't found in the cache so look on the file system. - - final File file = new File(filterProfilesDirectory, filterProfileName + JSON_EXTENSION); - - if (file.exists()) { - - filterProfileJson = FileUtils.readFileToString(file, Charset.defaultCharset()); - - // Put it in the cache. - filterProfileCacheService.insert(filterProfileName, filterProfileJson); - - } else { - throw new FileNotFoundException("Filter profile [" + filterProfileName + "] does not exist."); - } - - } - - return filterProfileJson; - - } - - @Override - public Map getAll() throws IOException { - - final Map filterProfiles = new HashMap<>(); - - // Read the filter profiles from the file system. - final Collection files = FileUtils.listFiles(new File(filterProfilesDirectory), new String[]{"json"}, false); - LOGGER.info("Found {} filter profiles", files.size()); - - for (final File file : files) { - - LOGGER.info("Loading filter profile {}", file.getAbsolutePath()); - final String json = FileUtils.readFileToString(file, Charset.defaultCharset()); - - final JSONObject object = new JSONObject(json); - final String name = object.getString("name"); - - filterProfiles.put(name, json); - LOGGER.info("Added filter profile named [{}]", name); - - } - - return filterProfiles; - - } - - @Override - public void save(String filterProfileJson) throws IOException { - - try { - - final JSONObject object = new JSONObject(filterProfileJson); - final String filterProfileName = object.getString("name"); - - final File file = new File(filterProfilesDirectory, filterProfileName + JSON_EXTENSION); - - FileUtils.writeStringToFile(file, filterProfileJson, Charset.defaultCharset()); - - // Put this filter profile into the cache. - filterProfileCacheService.insert(filterProfileName, filterProfileJson); - - } catch (JSONException ex) { - - LOGGER.error("The provided filter profile is not valid.", ex); - throw new BadRequestException("The provided filter profile is not valid."); - - } - - } - - @Override - public void delete(String filterProfileName) throws IOException { - - final File file = new File(filterProfilesDirectory, filterProfileName + JSON_EXTENSION); - - if(file.exists()) { - - if(!file.delete()) { - throw new IOException("Unable to delete filter profile " + filterProfileName + JSON_EXTENSION); - } - - // Remove it from the cache. - filterProfileCacheService.remove(filterProfileName); - - } else { - throw new FileNotFoundException("Filter profile with name " + filterProfileName + " does not exist."); - } - - } - -} - - diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalPolicyService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalPolicyService.java new file mode 100644 index 00000000..c5641ceb --- /dev/null +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/LocalPolicyService.java @@ -0,0 +1,180 @@ +/* + * Copyright 2023 Philterd, LLC @ https://www.philterd.ai + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.philterd.phileas.services.profiles; + +import ai.philterd.phileas.configuration.PhileasConfiguration; +import ai.philterd.phileas.model.exceptions.api.BadRequestException; +import ai.philterd.phileas.model.services.AbstractPolicyService; +import ai.philterd.phileas.model.services.PolicyCacheService; +import ai.philterd.phileas.model.services.PolicyService; +import ai.philterd.phileas.services.profiles.cache.InMemoryPolicyCacheService; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.*; + +public class LocalPolicyService extends AbstractPolicyService implements PolicyService { + + private static final Logger LOGGER = LogManager.getLogger(LocalPolicyService.class); + + private static final String JSON_EXTENSION = ".json"; + + private String policiesDirectory; + private PolicyCacheService policyCacheService; + + public LocalPolicyService(PhileasConfiguration phileasConfiguration) { + + this.policiesDirectory = phileasConfiguration.policiesDirectory(); + LOGGER.info("Looking for policies in {}", policiesDirectory); + + // Always use an in-memory cache when using a local policy service. + this.policyCacheService = new InMemoryPolicyCacheService(); + + } + + @Override + public List get() throws IOException { + + // This function never uses a cache. + + final List names = new LinkedList<>(); + + // Read the policies from the file system. + final Collection files = FileUtils.listFiles(new File(policiesDirectory), new String[]{"json"}, false); + + for(final File file : files) { + + final String json = FileUtils.readFileToString(file, Charset.defaultCharset()); + + final JSONObject object = new JSONObject(json); + final String name = object.getString("name"); + + names.add(name); + + } + + return names; + + } + + @Override + public String get(String policyName) throws IOException { + + String policyJson = policyCacheService.get(policyName); + + if(policyJson == null) { + + // The policy wasn't found in the cache so look on the file system. + + final File file = new File(policiesDirectory, policyName + JSON_EXTENSION); + + if (file.exists()) { + + policyJson = FileUtils.readFileToString(file, Charset.defaultCharset()); + + // Put it in the cache. + policyCacheService.insert(policyName, policyJson); + + } else { + throw new FileNotFoundException("Filter profile [" + policyName + "] does not exist."); + } + + } + + return policyJson; + + } + + @Override + public Map getAll() throws IOException { + + final Map policies = new HashMap<>(); + + // Read the policies from the file system. + final Collection files = FileUtils.listFiles(new File(policiesDirectory), new String[]{"json"}, false); + LOGGER.info("Found {} policies", files.size()); + + for (final File file : files) { + + LOGGER.info("Loading policy {}", file.getAbsolutePath()); + final String json = FileUtils.readFileToString(file, Charset.defaultCharset()); + + final JSONObject object = new JSONObject(json); + final String name = object.getString("name"); + + policies.put(name, json); + LOGGER.info("Added policy named [{}]", name); + + } + + return policies; + + } + + @Override + public void save(String policyJson) throws IOException { + + try { + + final JSONObject object = new JSONObject(policyJson); + final String policyName = object.getString("name"); + + final File file = new File(policiesDirectory, policyName + JSON_EXTENSION); + + FileUtils.writeStringToFile(file, policyJson, Charset.defaultCharset()); + + // Put this policy into the cache. + policyCacheService.insert(policyName, policyJson); + + } catch (JSONException ex) { + + LOGGER.error("The provided policy is not valid.", ex); + throw new BadRequestException("The provided policy is not valid."); + + } + + } + + @Override + public void delete(String policyName) throws IOException { + + final File file = new File(policiesDirectory, policyName + JSON_EXTENSION); + + if(file.exists()) { + + if(!file.delete()) { + throw new IOException("Unable to delete policy " + policyName + JSON_EXTENSION); + } + + // Remove it from the cache. + policyCacheService.remove(policyName); + + } else { + throw new FileNotFoundException("Filter profile with name " + policyName + " does not exist."); + } + + } + +} + + diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3FilterProfileService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3PolicyService.java similarity index 64% rename from phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3FilterProfileService.java rename to phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3PolicyService.java index 158102eb..08635e21 100644 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3FilterProfileService.java +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/S3PolicyService.java @@ -25,10 +25,10 @@ import ai.philterd.phileas.configuration.PhileasConfiguration; import ai.philterd.phileas.model.exceptions.api.BadRequestException; import ai.philterd.phileas.model.exceptions.api.InternalServerErrorException; -import ai.philterd.phileas.model.services.AbstractFilterProfileService; -import ai.philterd.phileas.model.services.FilterProfileCacheService; -import ai.philterd.phileas.model.services.FilterProfileService; -import ai.philterd.phileas.services.profiles.cache.FilterProfileCacheServiceFactory; +import ai.philterd.phileas.model.services.AbstractPolicyService; +import ai.philterd.phileas.model.services.PolicyCacheService; +import ai.philterd.phileas.model.services.PolicyService; +import ai.philterd.phileas.services.profiles.cache.PolicyCacheServiceFactory; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -41,35 +41,35 @@ import java.util.*; /** - * Implementation of {@link FilterProfileService} that is backed by S3. + * Implementation of {@link PolicyService} that is backed by S3. */ -public class S3FilterProfileService extends AbstractFilterProfileService implements FilterProfileService { +public class S3PolicyService extends AbstractPolicyService implements PolicyService { - private static final Logger LOGGER = LogManager.getLogger(S3FilterProfileService.class); + private static final Logger LOGGER = LogManager.getLogger(S3PolicyService.class); private static final String JSON_EXTENSION = ".json"; private AmazonS3 s3Client; private String bucket; private String prefix; - private FilterProfileCacheService filterProfileCacheService; + private PolicyCacheService policyCacheService; - public S3FilterProfileService(PhileasConfiguration phileasConfiguration, boolean testing) throws IOException { + public S3PolicyService(PhileasConfiguration phileasConfiguration, boolean testing) throws IOException { - this.bucket = phileasConfiguration.filterProfilesS3Bucket(); - final String region = phileasConfiguration.filterProfilesS3Region(); + this.bucket = phileasConfiguration.policiesS3Bucket(); + final String region = phileasConfiguration.policiesS3Region(); // If the prefix is not empty it must end with a forward slash. - if(!StringUtils.isEmpty(phileasConfiguration.filterProfilesS3Prefix())) { - if(!phileasConfiguration.filterProfilesS3Prefix().endsWith("/")) { - this.prefix = phileasConfiguration.filterProfilesS3Prefix() + "/"; + if(!StringUtils.isEmpty(phileasConfiguration.policiesS3Prefix())) { + if(!phileasConfiguration.policiesS3Prefix().endsWith("/")) { + this.prefix = phileasConfiguration.policiesS3Prefix() + "/"; } } - // Create a filter profile cache. - this.filterProfileCacheService = FilterProfileCacheServiceFactory.getInstance(phileasConfiguration); + // Create a policy cache. + this.policyCacheService = PolicyCacheServiceFactory.getInstance(phileasConfiguration); - LOGGER.info("Configuring S3 backend for filter profiles in s3 bucket {}", bucket); + LOGGER.info("Configuring S3 backend for policies in s3 bucket {}", bucket); if(testing) { @@ -100,11 +100,11 @@ public S3FilterProfileService(PhileasConfiguration phileasConfiguration, boolean @Override public List get() { - final List filterProfileNames = new LinkedList<>(); + final List policyNames = new LinkedList<>(); try { - LOGGER.info("Looking for filter profiles in s3 bucket {}", bucket); + LOGGER.info("Looking for policies in s3 bucket {}", bucket); final ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() .withBucketName(bucket) .withPrefix(prefix); @@ -125,8 +125,8 @@ public List get() { final JSONObject object = new JSONObject(json); final String name = object.getString("name"); - filterProfileNames.add(name); - LOGGER.debug("Found filter profile named {}", name); + policyNames.add(name); + LOGGER.debug("Found policy named {}", name); } @@ -139,36 +139,36 @@ public List get() { } catch (Exception ex) { - LOGGER.error("Unable to get filter profile names.", ex); + LOGGER.error("Unable to get policy names.", ex); - throw new InternalServerErrorException("Unable to get filter profile names."); + throw new InternalServerErrorException("Unable to get policy names."); } - return filterProfileNames; + return policyNames; } @Override - public String get(String filterProfileName) { + public String get(String policyName) { try { // Get from cache. - LOGGER.info("Getting profile name [{}] from the cache.", filterProfileName); - String json = filterProfileCacheService.get(filterProfileName); + LOGGER.info("Getting profile name [{}] from the cache.", policyName); + String json = policyCacheService.get(policyName); if(json == null) { - // The filter profile was not in the cache. Look in S3. - LOGGER.info("Filter profile was not cached. Looking for filter profile {} in s3 bucket {}", filterProfileName, bucket); - final S3Object fullObject = s3Client.getObject(new GetObjectRequest(bucket, prefix + filterProfileName + JSON_EXTENSION)); + // The policy was not in the cache. Look in S3. + LOGGER.info("Filter profile was not cached. Looking for policy {} in s3 bucket {}", policyName, bucket); + final S3Object fullObject = s3Client.getObject(new GetObjectRequest(bucket, prefix + policyName + JSON_EXTENSION)); json = IOUtils.toString(fullObject.getObjectContent(), StandardCharsets.UTF_8.name()); fullObject.close(); // Put it in the cache. - LOGGER.info("Caching filter profile [{}]", filterProfileName); - filterProfileCacheService.insert(filterProfileName, json); + LOGGER.info("Caching policy [{}]", policyName); + policyCacheService.insert(policyName, json); } @@ -176,9 +176,9 @@ public String get(String filterProfileName) { } catch (Exception ex) { - LOGGER.error("Unable to get filter profile.", ex); + LOGGER.error("Unable to get policy.", ex); - throw new InternalServerErrorException("Unable to get filter profile."); + throw new InternalServerErrorException("Unable to get policy."); } @@ -187,11 +187,11 @@ public String get(String filterProfileName) { @Override public Map getAll() { - final Map filterProfiles = new HashMap<>(); + final Map policies = new HashMap<>(); try { - LOGGER.info("Looking for all filter profiles in s3 bucket {}", bucket); + LOGGER.info("Looking for all policies in s3 bucket {}", bucket); final ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() .withBucketName(bucket) .withPrefix(prefix); @@ -202,7 +202,7 @@ public Map getAll() { result = s3Client.listObjectsV2(listObjectsV2Request); - LOGGER.info("Found {} filter profiles.", result.getObjectSummaries().size()); + LOGGER.info("Found {} policies.", result.getObjectSummaries().size()); for (final S3ObjectSummary objectSummary : result.getObjectSummaries()) { @@ -217,11 +217,11 @@ public Map getAll() { final JSONObject object = new JSONObject(json); final String name = object.getString("name"); - LOGGER.info("Adding filter profile named [{}]", name); - filterProfiles.put(name, json); + LOGGER.info("Adding policy named [{}]", name); + policies.put(name, json); - LOGGER.info("Caching filter profile [{}]", name); - filterProfileCacheService.insert(name, json); + LOGGER.info("Caching policy [{}]", name); + policyCacheService.insert(name, json); } @@ -236,61 +236,61 @@ public Map getAll() { } catch (Exception ex) { - LOGGER.error("Unable to get all filter profile names.", ex); + LOGGER.error("Unable to get all policy names.", ex); - throw new InternalServerErrorException("Unable to get all filter profiles."); + throw new InternalServerErrorException("Unable to get all policies."); } - return filterProfiles; + return policies; } @Override - public void save(String filterProfileJson) { + public void save(String policyJson) { try { - final JSONObject object = new JSONObject(filterProfileJson); + final JSONObject object = new JSONObject(policyJson); final String name = object.getString("name"); LOGGER.info("Uploading object to s3://{}/{}", bucket, name + JSON_EXTENSION); - s3Client.putObject(bucket, prefix + name + JSON_EXTENSION, filterProfileJson); + s3Client.putObject(bucket, prefix + name + JSON_EXTENSION, policyJson); // Insert it into the cache. - filterProfileCacheService.insert(name, filterProfileJson); + policyCacheService.insert(name, policyJson); } catch (JSONException ex) { - LOGGER.error("The provided filter profile is not valid.", ex); - throw new BadRequestException("The provided filter profile is not valid."); + LOGGER.error("The provided policy is not valid.", ex); + throw new BadRequestException("The provided policy is not valid."); } catch (Exception ex) { - LOGGER.error("Unable to save filter profile.", ex); + LOGGER.error("Unable to save policy.", ex); - throw new InternalServerErrorException("Unable to save filter profile."); + throw new InternalServerErrorException("Unable to save policy."); } } @Override - public void delete(String filterProfileName) { + public void delete(String policyName) { try { - LOGGER.info("Deleting object from s3://{}/{}", bucket, filterProfileName + JSON_EXTENSION); - s3Client.deleteObject(bucket, prefix + filterProfileName + JSON_EXTENSION); + LOGGER.info("Deleting object from s3://{}/{}", bucket, policyName + JSON_EXTENSION); + s3Client.deleteObject(bucket, prefix + policyName + JSON_EXTENSION); // Remove it from the cache. - filterProfileCacheService.remove(filterProfileName); + policyCacheService.remove(policyName); } catch (Exception ex) { - LOGGER.error("Unable to delete filter profile.", ex); + LOGGER.error("Unable to delete policy.", ex); - throw new InternalServerErrorException("Unable to delete filter profile."); + throw new InternalServerErrorException("Unable to delete policy."); } diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticFilterProfileService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticFilterProfileService.java deleted file mode 100644 index c1f9c155..00000000 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticFilterProfileService.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2023 Philterd, LLC @ https://www.philterd.ai - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.philterd.phileas.services.profiles; - -import com.google.gson.Gson; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.services.FilterProfileService; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * An implementation of {@link FilterProfileService} that uses a static - * filter profile - */ -public class StaticFilterProfileService implements FilterProfileService { - - private static final Logger LOGGER = LogManager.getLogger(StaticFilterProfileService.class); - - private FilterProfile filterProfile; - private Gson gson; - - /** - * Creates a static filter. - * @param filterProfileJson The filter profile JSON string. - */ - public StaticFilterProfileService(String filterProfileJson) { - - this.gson = new Gson(); - this.filterProfile = gson.fromJson(filterProfileJson, FilterProfile.class); - - } - - /** - * Creates a static filter. - * @param filterProfile A {@link FilterProfile}. - */ - public StaticFilterProfileService(FilterProfile filterProfile) { - - this.filterProfile = filterProfile; - - } - - @Override - public List get() throws IOException { - return Collections.emptyList(); - } - - /** - * Gets the filter profile. Note that the filterProfileName argument - * is ignored since there is only a single filter profile. - * @param filterProfileName The name of the filter profile. This value is ignored. - * @return The filter profile. - */ - @Override - public String get(String filterProfileName) { - - // The filterProfileName does not matter. - // There is only one filter profile and it is returned. - LOGGER.debug("Retrieving filter profile {}", filterProfile); - - return gson.toJson(filterProfile); - - } - - /** - * Gets a map of filter profiles. Note that the map will always only contain - * a single filter profile. - * @return A map of filter profiles. - */ - @Override - public Map getAll() { - - final Map filterProfiles = new HashMap<>(); - - filterProfiles.put(filterProfile.getName(), gson.toJson(filterProfile)); - - return filterProfiles; - - } - - @Override - public void save(String filterProfileJson) throws IOException { - // Will not be implemented. - } - - @Override - public void delete(String filterProfileName) throws IOException { - // Will not be implemented. - } - -} diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticPolicyService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticPolicyService.java new file mode 100644 index 00000000..29590f8b --- /dev/null +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/StaticPolicyService.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 Philterd, LLC @ https://www.philterd.ai + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.philterd.phileas.services.profiles; + +import com.google.gson.Gson; +import ai.philterd.phileas.model.profile.Policy; +import ai.philterd.phileas.model.services.PolicyService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * An implementation of {@link PolicyService} that uses a static + * policy + */ +public class StaticPolicyService implements PolicyService { + + private static final Logger LOGGER = LogManager.getLogger(StaticPolicyService.class); + + private Policy policy; + private Gson gson; + + /** + * Creates a static filter. + * @param policyJson The policy JSON string. + */ + public StaticPolicyService(String policyJson) { + + this.gson = new Gson(); + this.policy = gson.fromJson(policyJson, Policy.class); + + } + + /** + * Creates a static filter. + * @param policy A {@link Policy}. + */ + public StaticPolicyService(Policy policy) { + + this.policy = policy; + + } + + @Override + public List get() throws IOException { + return Collections.emptyList(); + } + + /** + * Gets the policy. Note that the policyName argument + * is ignored since there is only a single policy. + * @param policyName The name of the policy. This value is ignored. + * @return The policy. + */ + @Override + public String get(String policyName) { + + // The policyName does not matter. + // There is only one policy and it is returned. + LOGGER.debug("Retrieving policy {}", policy); + + return gson.toJson(policy); + + } + + /** + * Gets a map of policies. Note that the map will always only contain + * a single policy. + * @return A map of policies. + */ + @Override + public Map getAll() { + + final Map policies = new HashMap<>(); + + policies.put(policy.getName(), gson.toJson(policy)); + + return policies; + + } + + @Override + public void save(String policyJson) throws IOException { + // Will not be implemented. + } + + @Override + public void delete(String policyName) throws IOException { + // Will not be implemented. + } + +} diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryFilterProfileCacheService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryPolicyCacheService.java similarity index 70% rename from phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryFilterProfileCacheService.java rename to phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryPolicyCacheService.java index b6d2619d..47d5c369 100644 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryFilterProfileCacheService.java +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/InMemoryPolicyCacheService.java @@ -15,7 +15,7 @@ */ package ai.philterd.phileas.services.profiles.cache; -import ai.philterd.phileas.model.services.FilterProfileCacheService; +import ai.philterd.phileas.model.services.PolicyCacheService; import java.io.IOException; import java.util.ArrayList; @@ -23,11 +23,11 @@ import java.util.List; import java.util.Map; -public class InMemoryFilterProfileCacheService implements FilterProfileCacheService { +public class InMemoryPolicyCacheService implements PolicyCacheService { private Map cache; - public InMemoryFilterProfileCacheService() { + public InMemoryPolicyCacheService() { this.cache = new HashMap<>(); } @@ -37,8 +37,8 @@ public List get() throws IOException { } @Override - public String get(String filterProfileName) throws IOException { - return cache.get(filterProfileName); + public String get(String policyName) throws IOException { + return cache.get(policyName); } @Override @@ -47,13 +47,13 @@ public Map getAll() throws IOException { } @Override - public void insert(String filterProfileName, String filterProfile) { - cache.put(filterProfileName, filterProfile); + public void insert(String policyName, String policy) { + cache.put(policyName, policy); } @Override - public void remove(String filterProfileName) { - cache.remove(filterProfileName); + public void remove(String policyName) { + cache.remove(policyName); } @Override diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/FilterProfileCacheServiceFactory.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/PolicyCacheServiceFactory.java similarity index 68% rename from phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/FilterProfileCacheServiceFactory.java rename to phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/PolicyCacheServiceFactory.java index 638679fd..09952b27 100644 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/FilterProfileCacheServiceFactory.java +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/PolicyCacheServiceFactory.java @@ -16,25 +16,25 @@ package ai.philterd.phileas.services.profiles.cache; import ai.philterd.phileas.configuration.PhileasConfiguration; -import ai.philterd.phileas.model.services.FilterProfileCacheService; +import ai.philterd.phileas.model.services.PolicyCacheService; import java.io.IOException; -public class FilterProfileCacheServiceFactory { +public class PolicyCacheServiceFactory { - private FilterProfileCacheServiceFactory() { + private PolicyCacheServiceFactory() { // Use the static methods. } - public static FilterProfileCacheService getInstance(PhileasConfiguration phileasConfiguration) throws IOException { + public static PolicyCacheService getInstance(PhileasConfiguration phileasConfiguration) throws IOException { if(phileasConfiguration.cacheRedisEnabled()) { - return new RedisFilterProfileCacheService(phileasConfiguration); + return new RedisPolicyCacheService(phileasConfiguration); } else { - return new InMemoryFilterProfileCacheService(); + return new InMemoryPolicyCacheService(); } diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisFilterProfileCacheService.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisPolicyCacheService.java similarity index 72% rename from phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisFilterProfileCacheService.java rename to phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisPolicyCacheService.java index 73ac30bd..653fbbb6 100644 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisFilterProfileCacheService.java +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/cache/RedisPolicyCacheService.java @@ -17,7 +17,7 @@ import ai.philterd.phileas.configuration.PhileasConfiguration; import ai.philterd.phileas.model.cache.AbstractRedisCacheService; -import ai.philterd.phileas.model.services.FilterProfileCacheService; +import ai.philterd.phileas.model.services.PolicyCacheService; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.redisson.api.RMap; @@ -28,16 +28,16 @@ import java.util.List; import java.util.Map; -public class RedisFilterProfileCacheService extends AbstractRedisCacheService implements FilterProfileCacheService { +public class RedisPolicyCacheService extends AbstractRedisCacheService implements PolicyCacheService { - private static final Logger LOGGER = LogManager.getLogger(RedisFilterProfileCacheService.class); + private static final Logger LOGGER = LogManager.getLogger(RedisPolicyCacheService.class); - private static final String CACHE_KEY = "filterProfiles"; + private static final String CACHE_KEY = "policies"; - public RedisFilterProfileCacheService(PhileasConfiguration phileasConfiguration) throws IOException { + public RedisPolicyCacheService(PhileasConfiguration phileasConfiguration) throws IOException { super(phileasConfiguration); - LOGGER.info("Initializing Reds filter profile cache."); + LOGGER.info("Initializing Reds policy cache."); } @Override @@ -51,12 +51,12 @@ public List get() { } @Override - public String get(String filterProfileName) { + public String get(String policyName) { - // Get the filter profile from the cache and return it. + // Get the policy from the cache and return it. final RMap map = redisson.getMap(CACHE_KEY); - return map.get(filterProfileName); + return map.get(policyName); } @@ -67,7 +67,7 @@ public Map getAll() { if(count != 0) { - // Get the filter profiles from the cache and return them. + // Get the policies from the cache and return them. final RMap map = redisson.getMap(CACHE_KEY); Map m = new HashMap<>(); @@ -86,20 +86,20 @@ public Map getAll() { } @Override - public void insert(String filterProfileName, String filterProfile) { + public void insert(String policyName, String policy) { // Insert into the the cache. final RMap map = redisson.getMap(CACHE_KEY); - map.put(filterProfileName, filterProfile); + map.put(policyName, policy); } @Override - public void remove(String filterProfileName) { + public void remove(String policyName) { // Remove from the cache. final RMap map = redisson.getMap(CACHE_KEY); - map.remove(filterProfileName); + map.remove(policyName); } diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/FilterProfileUtils.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/FilterProfileUtils.java deleted file mode 100644 index b4551ba4..00000000 --- a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/FilterProfileUtils.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2023 Philterd, LLC @ https://www.philterd.ai - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.philterd.phileas.services.profiles.utils; - -import com.google.gson.Gson; -import ai.philterd.phileas.model.enums.FilterType; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.services.FilterProfileService; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.util.List; - -public class FilterProfileUtils { - - private static final Logger LOGGER = LogManager.getLogger(FilterProfileUtils.class); - - private FilterProfileService filterProfileService; - private Gson gson; - - public FilterProfileUtils(FilterProfileService filterProfileService, Gson gson) { - - this.filterProfileService = filterProfileService; - this.gson = gson; - - } - - public FilterProfile getCombinedFilterProfiles(final List filterProfileNames) throws IOException, IllegalStateException { - - // Get the deserialized filter profile of the first profile in the list. - // By starting off with a full profile we don't have to worry about adding - // Config and Crypto (and other sections) since those will always be - // taken from the first profile. - final FilterProfile combinedFilterProfile = getFilterProfile(filterProfileNames.get(0)); - - // In some chases there may be only one filter profile. - if(filterProfileNames.size() > 1) { - - // The name has no bearing on the results. We just want to give it a name. - combinedFilterProfile.setName("combined"); - - // Loop over the filter profile names and skip the first one since we have already - // deserialized it to a filter profile a few lines above. - for (final String filterProfileName : filterProfileNames.subList(1, filterProfileNames.size())) { - - // Get the filter profile. - final FilterProfile filterProfile = getFilterProfile(filterProfileName); - - // For each of the filter types, copy the filter (if it exists) from the source filter profile - // to the destination (combined) filter profile. If a filter already exists in the destination (combined) - // filter profile then throw an error. - - for(FilterType filterType : FilterType.values()) { - if (filterProfile.getIdentifiers().hasFilter(filterType)) { - if (!combinedFilterProfile.getIdentifiers().hasFilter(filterType)) { - combinedFilterProfile.getIdentifiers().setFilter(filterType, filterProfile.getIdentifiers().getFilter(filterType)); - } else { - throw new IllegalStateException("Filter profile has duplicate filter: " + filterType.toString()); - } - } - } - - // Aggregate the Ignored and IgnoredPatterns into the combined profile. - combinedFilterProfile.getIgnored().addAll(filterProfile.getIgnored()); - combinedFilterProfile.getIgnoredPatterns().addAll(filterProfile.getIgnoredPatterns()); - - } - - // The Config and Crypto sections are expected to be in the first filter profile. - // The Config and Crypto sections in the other filter profiles are ignored. - - // As other sections are added to the filter profile they will need added here, too. - - } - - return combinedFilterProfile; - - } - - private FilterProfile getFilterProfile(String filterProfileName) throws IOException { - - // This will ALWAYS return a filter profile because if it is not in the cache it will be retrieved from the cache. - // TODO: How to trigger a reload if the profile had to be retrieved from disk? - final String filterProfileJson = filterProfileService.get(filterProfileName); - - LOGGER.debug("Deserializing filter profile [{}]", filterProfileName); - return gson.fromJson(filterProfileJson, FilterProfile.class); - - } - -} diff --git a/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/PolicyUtils.java b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/PolicyUtils.java new file mode 100644 index 00000000..4531c2ce --- /dev/null +++ b/phileas-services/phileas-services-profiles/src/main/java/ai/philterd/phileas/services/profiles/utils/PolicyUtils.java @@ -0,0 +1,105 @@ +/* + * Copyright 2023 Philterd, LLC @ https://www.philterd.ai + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.philterd.phileas.services.profiles.utils; + +import com.google.gson.Gson; +import ai.philterd.phileas.model.enums.FilterType; +import ai.philterd.phileas.model.profile.Policy; +import ai.philterd.phileas.model.services.PolicyService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.List; + +public class PolicyUtils { + + private static final Logger LOGGER = LogManager.getLogger(PolicyUtils.class); + + private PolicyService policyService; + private Gson gson; + + public PolicyUtils(PolicyService policyService, Gson gson) { + + this.policyService = policyService; + this.gson = gson; + + } + + public Policy getCombinedPolicys(final List policyNames) throws IOException, IllegalStateException { + + // Get the deserialized policy of the first profile in the list. + // By starting off with a full profile we don't have to worry about adding + // Config and Crypto (and other sections) since those will always be + // taken from the first profile. + final Policy combinedPolicy = getPolicy(policyNames.get(0)); + + // In some chases there may be only one policy. + if(policyNames.size() > 1) { + + // The name has no bearing on the results. We just want to give it a name. + combinedPolicy.setName("combined"); + + // Loop over the policy names and skip the first one since we have already + // deserialized it to a policy a few lines above. + for (final String policyName : policyNames.subList(1, policyNames.size())) { + + // Get the policy. + final Policy policy = getPolicy(policyName); + + // For each of the filter types, copy the filter (if it exists) from the source policy + // to the destination (combined) policy. If a filter already exists in the destination (combined) + // policy then throw an error. + + for(FilterType filterType : FilterType.values()) { + if (policy.getIdentifiers().hasFilter(filterType)) { + if (!combinedPolicy.getIdentifiers().hasFilter(filterType)) { + combinedPolicy.getIdentifiers().setFilter(filterType, policy.getIdentifiers().getFilter(filterType)); + } else { + throw new IllegalStateException("Filter profile has duplicate filter: " + filterType.toString()); + } + } + } + + // Aggregate the Ignored and IgnoredPatterns into the combined profile. + combinedPolicy.getIgnored().addAll(policy.getIgnored()); + combinedPolicy.getIgnoredPatterns().addAll(policy.getIgnoredPatterns()); + + } + + // The Config and Crypto sections are expected to be in the first policy. + // The Config and Crypto sections in the other policies are ignored. + + // As other sections are added to the policy they will need added here, too. + + } + + return combinedPolicy; + + } + + private Policy getPolicy(String policyName) throws IOException { + + // This will ALWAYS return a policy because if it is not in the cache it will be retrieved from the cache. + // TODO: How to trigger a reload if the profile had to be retrieved from disk? + final String policyJson = policyService.get(policyName); + + LOGGER.debug("Deserializing policy [{}]", policyName); + return gson.fromJson(policyJson, Policy.class); + + } + +} diff --git a/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/FilterProfileUtilsTest.java b/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/FilterProfileUtilsTest.java deleted file mode 100644 index 7826630e..00000000 --- a/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/FilterProfileUtilsTest.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright 2023 Philterd, LLC @ https://www.philterd.ai - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.philterd.test.phileas.services.profiles.utils; - -import com.google.gson.Gson; -import ai.philterd.phileas.model.enums.FilterType; -import ai.philterd.phileas.model.profile.FilterProfile; -import ai.philterd.phileas.model.services.FilterProfileService; -import ai.philterd.phileas.services.profiles.utils.FilterProfileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Arrays; - -import static org.mockito.Mockito.when; - -public class FilterProfileUtilsTest { - - @Disabled - @Test - public void onlyOne() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile1")).thenReturn(json1); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile1")); - - final FilterProfile originalFilterProfile = gson.fromJson(json1, FilterProfile.class); - - // TODO: This needs a deep comparison. - Assertions.assertTrue(originalFilterProfile.equals(filterProfile)); - - Assertions.assertNotNull(filterProfile); - Assertions.assertFalse(StringUtils.equals(filterProfile.getName(), "combined")); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.AGE)); - Assertions.assertFalse(filterProfile.getIdentifiers().hasFilter(FilterType.CREDIT_CARD)); - Assertions.assertFalse(filterProfile.getIdentifiers().hasFilter(FilterType.URL)); - - } - - @Test - public void combineAgeAndCreditCard() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile2.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile1")).thenReturn(json1); - when(filterProfileService.get("profile2")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile1", "profile2")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertTrue(StringUtils.equals(filterProfile.getName(), "combined")); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.AGE)); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.CREDIT_CARD)); - Assertions.assertFalse(filterProfile.getIdentifiers().hasFilter(FilterType.URL)); - - } - - @Test - public void combineDuplicateFilter() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile1")).thenReturn(json1); - when(filterProfileService.get("profile2")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - - Assertions.assertThrows(IllegalStateException.class, () -> { - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile1", "profile2")); - }); - - } - - @Test - public void combineCustomDictionaryAndZipCode() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile3.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile4.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile3")).thenReturn(json1); - when(filterProfileService.get("profile4")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile3", "profile4")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - - @Test - public void combineWithCryptoInFirst() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile6.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile5")).thenReturn(json1); - when(filterProfileService.get("profile6")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile5", "profile6")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertNotNull(filterProfile.getCrypto()); - Assertions.assertTrue(StringUtils.equalsIgnoreCase(filterProfile.getCrypto().getKey(), "keyhere")); - Assertions.assertTrue(StringUtils.equalsIgnoreCase(filterProfile.getCrypto().getIv(), "ivhere")); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - - @Test - public void combineWithCryptoInSecond() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile6.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile6")).thenReturn(json1); - when(filterProfileService.get("profile5")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile6", "profile5")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertNull(filterProfile.getCrypto()); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - - @Test - public void combineWithConfigInFirst() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile7.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile7")).thenReturn(json1); - when(filterProfileService.get("profile5")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile7", "profile5")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertNull(filterProfile.getCrypto()); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - - @Test - public void combineWithIgnored() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile8.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile9.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile8")).thenReturn(json1); - when(filterProfileService.get("profile9")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile8", "profile9")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertNotNull(filterProfile.getIgnored()); - Assertions.assertEquals(2, filterProfile.getIgnored().size()); - Assertions.assertTrue(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - - @Test - public void combineWithIgnoredPatterns() throws IOException { - - final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile10.json"), Charset.defaultCharset()); - final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile11.json"), Charset.defaultCharset()); - - final FilterProfileService filterProfileService = Mockito.mock(FilterProfileService.class); - when(filterProfileService.get("profile10")).thenReturn(json1); - when(filterProfileService.get("profile11")).thenReturn(json2); - - final Gson gson = new Gson(); - final FilterProfileUtils filterProfileUtils = new FilterProfileUtils(filterProfileService, gson); - final FilterProfile filterProfile = filterProfileUtils.getCombinedFilterProfiles(Arrays.asList("profile10", "profile11")); - - Assertions.assertNotNull(filterProfile); - Assertions.assertNotNull(filterProfile.getIgnored()); - Assertions.assertEquals(2, filterProfile.getIgnoredPatterns().size()); - Assertions.assertFalse(filterProfile.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); - - } - -} diff --git a/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/PolicyUtilsTest.java b/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/PolicyUtilsTest.java new file mode 100644 index 00000000..5384a1f3 --- /dev/null +++ b/phileas-services/phileas-services-profiles/src/test/java/ai/philterd/test/phileas/services/profiles/utils/PolicyUtilsTest.java @@ -0,0 +1,229 @@ +/* + * Copyright 2023 Philterd, LLC @ https://www.philterd.ai + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.philterd.test.phileas.services.profiles.utils; + +import com.google.gson.Gson; +import ai.philterd.phileas.model.enums.FilterType; +import ai.philterd.phileas.model.profile.Policy; +import ai.philterd.phileas.model.services.PolicyService; +import ai.philterd.phileas.services.profiles.utils.PolicyUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Arrays; + +import static org.mockito.Mockito.when; + +public class PolicyUtilsTest { + + @Disabled + @Test + public void onlyOne() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile1")).thenReturn(json1); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile1")); + + final Policy originalPolicy = gson.fromJson(json1, Policy.class); + + // TODO: This needs a deep comparison. + Assertions.assertTrue(originalPolicy.equals(policy)); + + Assertions.assertNotNull(policy); + Assertions.assertFalse(StringUtils.equals(policy.getName(), "combined")); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.AGE)); + Assertions.assertFalse(policy.getIdentifiers().hasFilter(FilterType.CREDIT_CARD)); + Assertions.assertFalse(policy.getIdentifiers().hasFilter(FilterType.URL)); + + } + + @Test + public void combineAgeAndCreditCard() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile2.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile1")).thenReturn(json1); + when(policyService.get("profile2")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile1", "profile2")); + + Assertions.assertNotNull(policy); + Assertions.assertTrue(StringUtils.equals(policy.getName(), "combined")); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.AGE)); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.CREDIT_CARD)); + Assertions.assertFalse(policy.getIdentifiers().hasFilter(FilterType.URL)); + + } + + @Test + public void combineDuplicateFilter() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile1.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile1")).thenReturn(json1); + when(policyService.get("profile2")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + + Assertions.assertThrows(IllegalStateException.class, () -> { + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile1", "profile2")); + }); + + } + + @Test + public void combineCustomDictionaryAndZipCode() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile3.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile4.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile3")).thenReturn(json1); + when(policyService.get("profile4")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile3", "profile4")); + + Assertions.assertNotNull(policy); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.CUSTOM_DICTIONARY)); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + + @Test + public void combineWithCryptoInFirst() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile6.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile5")).thenReturn(json1); + when(policyService.get("profile6")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile5", "profile6")); + + Assertions.assertNotNull(policy); + Assertions.assertNotNull(policy.getCrypto()); + Assertions.assertTrue(StringUtils.equalsIgnoreCase(policy.getCrypto().getKey(), "keyhere")); + Assertions.assertTrue(StringUtils.equalsIgnoreCase(policy.getCrypto().getIv(), "ivhere")); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + + @Test + public void combineWithCryptoInSecond() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile6.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile6")).thenReturn(json1); + when(policyService.get("profile5")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile6", "profile5")); + + Assertions.assertNotNull(policy); + Assertions.assertNull(policy.getCrypto()); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + + @Test + public void combineWithConfigInFirst() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile7.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile5.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile7")).thenReturn(json1); + when(policyService.get("profile5")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile7", "profile5")); + + Assertions.assertNotNull(policy); + Assertions.assertNull(policy.getCrypto()); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + + @Test + public void combineWithIgnored() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile8.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile9.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile8")).thenReturn(json1); + when(policyService.get("profile9")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile8", "profile9")); + + Assertions.assertNotNull(policy); + Assertions.assertNotNull(policy.getIgnored()); + Assertions.assertEquals(2, policy.getIgnored().size()); + Assertions.assertTrue(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + + @Test + public void combineWithIgnoredPatterns() throws IOException { + + final String json1 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile10.json"), Charset.defaultCharset()); + final String json2 = IOUtils.toString(this.getClass().getResourceAsStream("/profiles/profile11.json"), Charset.defaultCharset()); + + final PolicyService policyService = Mockito.mock(PolicyService.class); + when(policyService.get("profile10")).thenReturn(json1); + when(policyService.get("profile11")).thenReturn(json2); + + final Gson gson = new Gson(); + final PolicyUtils policyUtils = new PolicyUtils(policyService, gson); + final Policy policy = policyUtils.getCombinedPolicys(Arrays.asList("profile10", "profile11")); + + Assertions.assertNotNull(policy); + Assertions.assertNotNull(policy.getIgnored()); + Assertions.assertEquals(2, policy.getIgnoredPatterns().size()); + Assertions.assertFalse(policy.getIdentifiers().hasFilter(FilterType.ZIP_CODE)); + + } + +}