Skip to content

Commit

Permalink
Refactor output strategy + system tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@AleDore
AleDore committed Mar 1, 2024
1 parent baf816f commit 8baacc9
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 99 deletions.
69 changes: 51 additions & 18 deletions __integrations__/__tests__/system/deduplication/table.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as E from "fp-ts/Either";
import * as TE from "fp-ts/lib/TaskEither";
import { pipe } from "fp-ts/lib/function";
import {
IOutputDocument,
createIndexIfNotExists,
getElasticClient,
} from "../../../../src/output/elasticsearch/elasticsearch";
Expand All @@ -14,33 +15,38 @@ import {
} from "../../../../src/output/factory";
import { ELASTIC_NODE, STORAGE_CONN_STRING } from "../../../env";
import {
createTable,
deleteTable,
getTableClient,
getTableDocument,
} from "../../../../src/utils/tableStorage";
import { deleteData, deleteIndex } from "../../../utils/elasticsearch";
import { deleteIndex } from "../../../utils/elasticsearch";
import { createTableIfNotExists, deleteTableWithAbort } from "../../../utils/table";
import { NonEmptyString } from "@pagopa/ts-commons/lib/strings";

const INDEX_NAME = "index";
const INDEX_NAME = "index" as NonEmptyString;
const FIRST_ID = "first_id";

const currentTimestamp = Date.now();
const oldTimestamp = new Date(currentTimestamp - 86400000).getTime();

const newerDocument = {
id: FIRST_ID,
const getNewerDocument = (id: string = FIRST_ID) => ({
id,
_timestamp: currentTimestamp,
value: "first",
};
});

const olderDocument = {
id: FIRST_ID,
const getOlderDocument = (id: string = FIRST_ID) => ({
id,
_timestamp: oldTimestamp,
value: "first",
};
});

const tableDeduplicationStrategyConfig: DeduplicationStrategyConfig = {
type: DeduplicationStrategyType.TableStorage,
tableName: INDEX_NAME,
storageConnectionString: STORAGE_CONN_STRING,
opts: {
allowInsecureConnection: true
}
};

describe("table deduplication", () => {
Expand All @@ -50,7 +56,7 @@ describe("table deduplication", () => {
TE.fromEither,
TE.chainFirst((client) => createIndexIfNotExists(client, INDEX_NAME)),
TE.chain(() =>
createTable(
createTableIfNotExists(
getTableClient(INDEX_NAME, { allowInsecureConnection: true })(
STORAGE_CONN_STRING,
),
Expand All @@ -63,25 +69,28 @@ describe("table deduplication", () => {
}),
)();
}, 10000);

afterAll(async () => {
await pipe(
getElasticClient(ELASTIC_NODE),
TE.fromEither,
TE.chainFirst((client) => deleteIndex(client, INDEX_NAME)),
TE.chain(() => deleteTable(
getTableClient(INDEX_NAME, { allowInsecureConnection: true })(
STORAGE_CONN_STRING,
TE.chain(() =>
deleteTableWithAbort(
getTableClient(INDEX_NAME, { allowInsecureConnection: true })(
STORAGE_CONN_STRING,
),
),
)),
),
TE.getOrElse((e) => {
throw Error(
`Cannot destroy integration tests data - ${JSON.stringify(e.message)}`,
);
}),
)();
}, 10000);
it("should create the document when it doesn't exists", async () => {
it("should create the document if it doesn't exists", async () => {
const olderDocument = getOlderDocument();
await pipe(
E.Do,
E.bind("service", () => getElasticSearchService(ELASTIC_NODE)),
Expand All @@ -97,7 +106,7 @@ describe("table deduplication", () => {
),
TE.bimap(
(err) =>
new Error(
Error(
`it should not fail while finding an existing index - ${err.message}`,
),
({ service }) =>
Expand All @@ -107,12 +116,35 @@ describe("table deduplication", () => {
TE.bimap(E.toError, (doc) =>
expect(doc._source).toEqual(olderDocument),
),
TE.chain(() =>
pipe(
getTableClient(INDEX_NAME, { allowInsecureConnection: true })(
STORAGE_CONN_STRING,
),
TE.of,
TE.chain((tableClient) =>
getTableDocument<IOutputDocument>(
tableClient,
INDEX_NAME,
olderDocument.id,
),
),
TE.chain(
TE.fromOption(() => Error("Table document should exists")),
),
TE.map((tableDoc) => expect(tableDoc).toEqual(olderDocument)),
),
),
),
),
TE.mapLeft((e) => {
throw e;
}),
)();
});

it("should update the document when it has a greater timestamp than the one in the index", async () => {
const newerDocument = getNewerDocument();
await pipe(
E.Do,
E.bind("service", () => getElasticSearchService(ELASTIC_NODE)),
Expand Down Expand Up @@ -144,6 +176,7 @@ describe("table deduplication", () => {
});

it("should not update the document when it has a lower timestamp than the one in the index", async () => {
const [olderDocument, newerDocument] = [getOlderDocument(), getNewerDocument()]
await pipe(
E.Do,
E.bind("service", () => getElasticSearchService(ELASTIC_NODE)),
Expand Down
4 changes: 3 additions & 1 deletion __integrations__/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
"keywords": [],
"author": "",
"scripts": {
"start": "docker-compose up -d",
"start": "docker-compose --env-file environments/.env up -d",
"stop": "docker-compose down",
"build": "tsc",
"test": "jest --runInBand"
},
"dependencies": {
"@azure/abort-controller": "^2.0.0",
"@azure/cosmos": "^4.0.0",
"@azure/data-tables": "^13.2.2",
"dotenv": "^16.3.1",
"fp-ts": "^2.16.1"
},
Expand Down
24 changes: 24 additions & 0 deletions __integrations__/utils/table.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import * as DT from "@azure/data-tables";
import * as E from "fp-ts/lib/Either";
import * as TE from "fp-ts/lib/TaskEither";
import { pipe } from "fp-ts/lib/function";

export const createTableIfNotExists = (
tableClient: DT.TableClient,
): TE.TaskEither<Error, void> =>
pipe(new AbortController(), (controller) =>
TE.tryCatch(
() => tableClient.createTable({ abortSignal: controller.signal }),
E.toError,
),
);

export const deleteTableWithAbort = (
tableClient: DT.TableClient,
): TE.TaskEither<Error, void> =>
pipe(new AbortController(), (controller) =>
TE.tryCatch(
() => tableClient.deleteTable({ abortSignal: controller.signal }),
E.toError,
),
);
84 changes: 84 additions & 0 deletions __integrations__/yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@
dependencies:
tslib "^2.2.0"

"@azure/abort-controller@^2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-2.0.0.tgz#a66d26c7f64977e3ff4b9e0b136296cb4bd47e8b"
integrity sha512-RP/mR/WJchR+g+nQFJGOec+nzeN/VvjlwbinccoqfhTsTHbb8X5+mLDp48kHT0ueyum0BNSwGm0kX0UZuIqTGg==
dependencies:
tslib "^2.2.0"

"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0":
version "1.5.0"
resolved "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.5.0.tgz#a41848c5c31cb3b7c84c409885267d55a2c92e44"
Expand All @@ -26,6 +33,40 @@
"@azure/core-util" "^1.1.0"
tslib "^2.2.0"

"@azure/core-client@^1.0.0":
version "1.8.0"
resolved "https://registry.yarnpkg.com/@azure/core-client/-/core-client-1.8.0.tgz#fce9b0af62ba469510e4ed6169b75622d31e2216"
integrity sha512-+gHS3gEzPlhyQBMoqVPOTeNH031R5DM/xpCvz72y38C09rg4Hui/1sJS/ujoisDZbbSHyuRLVWdFlwL0pIFwbg==
dependencies:
"@azure/abort-controller" "^2.0.0"
"@azure/core-auth" "^1.4.0"
"@azure/core-rest-pipeline" "^1.9.1"
"@azure/core-tracing" "^1.0.0"
"@azure/core-util" "^1.0.0"
"@azure/logger" "^1.0.0"
tslib "^2.2.0"

"@azure/core-paging@^1.1.1":
version "1.5.0"
resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.5.0.tgz#5a5b09353e636072e6a7fc38f7879e11d0afb15f"
integrity sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==
dependencies:
tslib "^2.2.0"

"@azure/core-rest-pipeline@^1.1.0", "@azure/core-rest-pipeline@^1.9.1":
version "1.14.0"
resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.14.0.tgz#9ff394941580a6dee9f0c8a759e16065c524bcfc"
integrity sha512-Tp4M6NsjCmn9L5p7HsW98eSOS7A0ibl3e5ntZglozT0XuD/0y6i36iW829ZbBq0qihlGgfaeFpkLjZ418KDm1Q==
dependencies:
"@azure/abort-controller" "^2.0.0"
"@azure/core-auth" "^1.4.0"
"@azure/core-tracing" "^1.0.1"
"@azure/core-util" "^1.3.0"
"@azure/logger" "^1.0.0"
http-proxy-agent "^5.0.0"
https-proxy-agent "^5.0.0"
tslib "^2.2.0"

"@azure/core-rest-pipeline@^1.2.0":
version "1.13.0"
resolved "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.13.0.tgz#770b003c351b4869e3f1c85800bacb947c98cd33"
Expand All @@ -47,6 +88,14 @@
dependencies:
tslib "^2.2.0"

"@azure/core-util@^1.0.0":
version "1.7.0"
resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.7.0.tgz#3a2f73e8c7eed0666e8b6ff9ca2c1951e175feba"
integrity sha512-Zq2i3QO6k9DA8vnm29mYM4G8IE9u1mhF1GUabVEqPNX8Lj833gdxQ2NAFxt2BZsfAL+e9cT8SyVN7dFVJ/Hf0g==
dependencies:
"@azure/abort-controller" "^2.0.0"
tslib "^2.2.0"

"@azure/core-util@^1.1.0", "@azure/core-util@^1.3.0":
version "1.6.1"
resolved "https://registry.npmjs.org/@azure/core-util/-/core-util-1.6.1.tgz#fea221c4fa43c26543bccf799beb30c1c7878f5a"
Expand All @@ -55,6 +104,14 @@
"@azure/abort-controller" "^1.0.0"
tslib "^2.2.0"

"@azure/core-xml@^1.0.0":
version "1.3.4"
resolved "https://registry.yarnpkg.com/@azure/core-xml/-/core-xml-1.3.4.tgz#516092f948b1b609b6c2afb8437acd204527e323"
integrity sha512-B1xI79Ur/u+KR69fGTcsMNj8KDjBSqAy0Ys6Byy4Qm1CqoUy7gCT5A7Pej0EBWRskuH6bpCwrAnosfmQEalkcg==
dependencies:
fast-xml-parser "^4.2.4"
tslib "^2.2.0"

"@azure/cosmos@^4.0.0":
version "4.0.0"
resolved "https://registry.npmjs.org/@azure/cosmos/-/cosmos-4.0.0.tgz#5fda8b35cb62bbcda52159b96c4c3981a843d5b9"
Expand All @@ -74,6 +131,21 @@
universal-user-agent "^6.0.0"
uuid "^8.3.0"

"@azure/data-tables@^13.2.2":
version "13.2.2"
resolved "https://registry.yarnpkg.com/@azure/data-tables/-/data-tables-13.2.2.tgz#9aa82992d7317a779ecf66436ffa96b887d8e616"
integrity sha512-Dq2Aq0mMMF0BPzYQKdBY/OtO7VemP/foh6z+mJpUO1hRL+65C1rGQUJf20LJHotSyU8wHb4HJzOs+Z50GXSy1w==
dependencies:
"@azure/core-auth" "^1.3.0"
"@azure/core-client" "^1.0.0"
"@azure/core-paging" "^1.1.1"
"@azure/core-rest-pipeline" "^1.1.0"
"@azure/core-tracing" "^1.0.0"
"@azure/core-xml" "^1.0.0"
"@azure/logger" "^1.0.0"
tslib "^2.2.0"
uuid "^8.3.0"

"@azure/logger@^1.0.0":
version "1.0.4"
resolved "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz#28bc6d0e5b3c38ef29296b32d35da4e483593fa1"
Expand Down Expand Up @@ -1144,6 +1216,13 @@ [email protected], fast-json-stable-stringify@^2.1.0:
resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==

fast-xml-parser@^4.2.4:
version "4.3.5"
resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.3.5.tgz#e2f2a2ae8377e9c3dc321b151e58f420ca7e5ccc"
integrity sha512-sWvP1Pl8H03B8oFJpFR3HE31HUfwtX7Rlf9BNsvdpujD4n7WMhfmu8h9wOV2u+c1k0ZilTADhPqypzx2J690ZQ==
dependencies:
strnum "^1.0.5"

fb-watchman@^2.0.0:
version "2.0.2"
resolved "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz#e9524ee6b5c77e9e5001af0f85f3adbb8623255c"
Expand Down Expand Up @@ -2174,6 +2253,11 @@ strip-json-comments@^3.1.1:
resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==

strnum@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==

supports-color@^5.3.0:
version "5.5.0"
resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
Expand Down
Loading

0 comments on commit 8baacc9

Please sign in to comment.