From 072e114b0a8c07c2a52513602d34da272d76377c Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Mon, 26 Jun 2023 21:21:24 +0300 Subject: [PATCH 01/13] ci for express --- .github/workflows/E2E_CI.yaml | 121 +++++++++++ Dockerfile | 31 +++ ci_files/build_cluster.sh | 66 ++++++ ci_files/extract_features.sh | 7 + ci_files/wait-for-job.sh | 42 ++++ demo-site/scripts/build_docker.sh | 161 ++++++++++++++ demo-site/scripts/clean_static_files.js | 58 +++++ demo-site/scripts/common.sh | 72 +++++++ demo-site/scripts/create_px_configs.js | 22 ++ demo-site/scripts/create_static_files.js | 105 +++++++++ demo-site/scripts/run_docker.sh | 62 ++++++ demo-site/servers/nodejs/app.js | 202 ++++++++++++++++++ demo-site/servers/nodejs/config.inc.json | 16 ++ demo-site/servers/nodejs/config.json | 23 ++ demo-site/servers/nodejs/package.json | 27 +++ demo-site/servers/nodejs/px_config.json | 87 ++++++++ demo-site/shared_config.json | 83 +++++++ demo-site/templates/origin/Dockerfile | 39 ++++ demo-site/templates/origin/origin_app.js | 73 +++++++ demo-site/templates/origin/package.json | 12 ++ .../static_files/index.template.html | 44 ++++ .../static_files/profile.template.html | 39 ++++ demo-site/templates/static_files/style.css | 115 ++++++++++ .../templates/test_endpoints/package.json | 12 ++ .../test_endpoints/test_endpoints_app.js | 94 ++++++++ demo-site/utils/cdn_deploy_tool_utils.js | 82 +++++++ demo-site/utils/constants.js | 55 +++++ demo-site/utils/utils.js | 115 ++++++++++ px_metadata.json | 17 ++ 29 files changed, 1882 insertions(+) create mode 100644 .github/workflows/E2E_CI.yaml create mode 100644 Dockerfile create mode 100755 ci_files/build_cluster.sh create mode 100644 ci_files/extract_features.sh create mode 100755 ci_files/wait-for-job.sh create mode 100755 demo-site/scripts/build_docker.sh create mode 100644 demo-site/scripts/clean_static_files.js create mode 100644 demo-site/scripts/common.sh create mode 100644 demo-site/scripts/create_px_configs.js create mode 100644 demo-site/scripts/create_static_files.js create mode 100755 demo-site/scripts/run_docker.sh create mode 100644 demo-site/servers/nodejs/app.js create mode 100644 demo-site/servers/nodejs/config.inc.json create mode 100644 demo-site/servers/nodejs/config.json create mode 100644 demo-site/servers/nodejs/package.json create mode 100644 demo-site/servers/nodejs/px_config.json create mode 100644 demo-site/shared_config.json create mode 100644 demo-site/templates/origin/Dockerfile create mode 100644 demo-site/templates/origin/origin_app.js create mode 100644 demo-site/templates/origin/package.json create mode 100644 demo-site/templates/static_files/index.template.html create mode 100644 demo-site/templates/static_files/profile.template.html create mode 100644 demo-site/templates/static_files/style.css create mode 100644 demo-site/templates/test_endpoints/package.json create mode 100644 demo-site/templates/test_endpoints/test_endpoints_app.js create mode 100644 demo-site/utils/cdn_deploy_tool_utils.js create mode 100644 demo-site/utils/constants.js create mode 100644 demo-site/utils/utils.js diff --git a/.github/workflows/E2E_CI.yaml b/.github/workflows/E2E_CI.yaml new file mode 100644 index 0000000..6732331 --- /dev/null +++ b/.github/workflows/E2E_CI.yaml @@ -0,0 +1,121 @@ +name: E2E Build + +on: + pull_request + +jobs: + + extract_metadata: + runs-on: ubuntu-latest + name: Extract supported_features + outputs: + supported-features: ${{ steps.supported-features.outputs.value }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '18.x' + - name: extract supported features + id: supported-features + run: echo "value=$(node -p -e "require('./px_metadata.json').supported_features?.join(' or ') || ''")" >> "$GITHUB_OUTPUT" + + + CI: + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: + - extract_metadata + + steps: + + - name: build local cluster + uses: actions/checkout@v2 + - run: ./ci_files/build_cluster.sh + + - name: Set up Docker + uses: docker/setup-buildx-action@v1 + + - name: Build Sample-site Docker image + run: | + docker build -t localhost:5001/node-sample-site:1.0.0 . && docker images && docker push localhost:5001/node-sample-site:1.0.0 + env: + DOCKER_BUILDKIT: 1 + + + - name: install helm + run: | + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + sudo apt-get install apt-transport-https --yes + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + sudo apt-get update + sudo apt-get install helm + + - name: Checkout enforcer repo + uses: actions/checkout@v2 + + - name: Clone helm charts repo + uses: actions/checkout@v2 + with: + repository: PerimeterX/connect-helm-charts + token: ${{ secrets.CONNECT_PULL_TOKEN }} + ref: main + path: ./deploy_charts + + + - name: deploy sample site + run: | + helm install sample-site ./deploy_charts/charts/sample-site --set image.name=localhost:5001/node-sample-site --set image.tag=1.0.0 --set imagePullPolicy=Always --set collectorURL=http://mock-collector-mock-collector:3001 --wait + + - name: Set up Google Cloud SDK + id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCR_SA_KEY }}' + + - name: Configure Docker credentials + run: | + gcloud auth configure-docker gcr.io + + - name: pull mock collector image + run: | + docker pull gcr.io/px-docker-repo/connecteam/mock-collector:1.0.2 && \ + docker tag gcr.io/px-docker-repo/connecteam/mock-collector:1.0.2 localhost:5001/mock-collector:1.0.2 && \ + docker push localhost:5001/mock-collector:1.0.2 && \ + docker images + + - name: deploy mock collector + run: | + helm install mock-collector ./deploy_charts/charts/mock-collector --set image.repository=localhost:5001/mock-collector --set image.tag=1.0.2 --set imagePullPolicy=Always --wait + + - run: kubectl get pods + + - name: pull enforcer tests image + run: | + docker pull gcr.io/px-docker-repo/connecteam/enforcer-specs-tests:1.1.0 && \ + docker tag gcr.io/px-docker-repo/connecteam/enforcer-specs-tests:1.1.0 localhost:5001/enforcer-spec-tests:1.1.0 && \ + docker push localhost:5001/enforcer-spec-tests:1.1.0 && \ + docker images + + - name: run enforcer tests + run: | + helm install enforcer-spec-tests ./deploy_charts/charts/enforcer-spec-tests --set image.repository=localhost:5001/enforcer-spec-tests --set image.tag=1.1.0 --set imagePullPolicy=Always \ + --set internalMockCollectorURL=http://mock-collector-mock-collector:3001 \ + --set appID=PXnEpdw6lS \ + --set siteURL=http://sample-site-sample-site:3000 \ + --set cookieSecret=${{ secrets.TEST_COOKIE_SECRET }} \ + --set supportedFeatures="${{ needs.extract_metadata.outputs.supported-features }}" \ + --set-file enforcerMetadataContent=./px_metadata.json + + - name: wait until test is over + run: ./ci_files/wait-for-job.sh + env: + JOB_NAME: enforcer-spec-tests + + - name: get tests results + if: ${{ failure() }} + run: kubectl logs job/enforcer-spec-tests + + - name: get tests results + run: kubectl logs job/enforcer-spec-tests \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c2d3d65 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# create static files and configs +FROM node:16-slim + +WORKDIR /workspace +COPY ./demo-site/shared_config.json . +COPY ./demo-site/scripts scripts +COPY ./demo-site/templates templates +COPY ./demo-site/utils utils +COPY ./demo-site/servers/nodejs/package.json servers/nodejs/package.json +RUN cd servers/nodejs && npm install +COPY ./demo-site/servers/nodejs servers/nodejs + +RUN node scripts/create_static_files.js && node scripts/create_px_configs.js + +WORKDIR /workspace/servers/nodejs + +COPY ./ perimeterx-node-express +RUN npm install ./perimeterx-node-express + +ARG ENABLE_TEST_ENDPOINTS=true +ARG PX_APP_ID="" +ARG PX_AUTH_TOKEN="" +ARG PX_COOKIE_SECRET="" + +ENV ENABLE_TEST_ENDPOINTS=${ENABLE_TEST_ENDPOINTS} +ENV PX_APP_ID=${PX_APP_ID} +ENV PX_AUTH_TOKEN=${PX_AUTH_TOKEN} +ENV PX_COOKIE_SECRET=${PX_COOKIE_SECRET} + +EXPOSE 3000 +CMD ["node","app.js"] diff --git a/ci_files/build_cluster.sh b/ci_files/build_cluster.sh new file mode 100755 index 0000000..5b87aa6 --- /dev/null +++ b/ci_files/build_cluster.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -o errexit + +# 1. Download kind binary +# For AMD64 / x86_64 +#[ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.19.0/kind-linux-amd64 +# For ARM64 +#[ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.19.0/kind-linux-arm64 +#chmod +x ./kind +#sudo mv ./kind /usr/local/bin/kind + + +# 2. Create registry container unless it already exists +reg_name='kind-registry' +reg_port='5001' +if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +# 3. Create kind cluster with containerd registry config dir enabled +cat <> /dev/null 2>&1 + success=$? + if [ $success -eq 0 ] + then + exit 0; + fi + + echo "checking for failure" + kubectl wait --for=condition=failed -n $ns job/$job --timeout=0s >> /dev/null 2>&1 + fail=$? + if [ $fail -eq 0 ] + then + exit 1 + fi + + sleep 5 +done diff --git a/demo-site/scripts/build_docker.sh b/demo-site/scripts/build_docker.sh new file mode 100755 index 0000000..83d6782 --- /dev/null +++ b/demo-site/scripts/build_docker.sh @@ -0,0 +1,161 @@ +#!/bin/bash +set -e + +function main { + initialize $@ + + copy_local_directories + + build_docker_image + + remove_tmp_enforcer_directories +} + +function initialize { + source ./scripts/common.sh + + validate_and_set_args $@ + + initialize_environment_variables +} + +function is_deploy_tool_required { + if [[ "$(is_cdn_enforcer $enforcer)" == "true" && $cdn_container_type != "origin" ]]; then + echo "true" + else + echo "false" + fi +} + +function is_enforcer_required { + if [[ "$(is_cdn_enforcer $enforcer)" == "true" && $cdn_container_type == "origin" ]]; then + echo "false" + else + echo "true" + fi +} + +function initialize_environment_variables { + echo "Initializing environment variables" + use_env_or_config_value PX_APP_ID px_app_id true + use_env_or_config_value PX_AUTH_TOKEN px_auth_token true + use_env_or_config_value PX_COOKIE_SECRET px_cookie_secret true + use_env_or_config_value DOCKER_ENFORCER_DIR docker_enforcer_dir + + if [[ "$(is_enforcer_required)" == "true" ]]; then + use_env_or_config_value LOCAL_ENFORCER_DIR local_enforcer_dir true + use_env_or_config_value LOCAL_CORE_ENFORCER_DIR local_core_enforcer_dir + fi + + if [[ "$(is_deploy_tool_required)" == "true" ]]; then + use_env_or_config_value LOCAL_DEPLOY_TOOL_DIR local_deploy_tool_dir true + fi +} + +function copy_local_directories { + export COPY_LOCAL_ENFORCER=${COPY_LOCAL_ENFORCER:-true} + export COPY_LOCAL_CORE_ENFORCER=${COPY_LOCAL_CORE_ENFORCER:-true} + export COPY_LOCAL_DEPLOY_TOOL_DIR=${COPY_LOCAL_DEPLOY_TOOL_DIR:-true} + + if [[ -n "${LOCAL_ENFORCER_DIR}" && "${COPY_LOCAL_ENFORCER}" == "true" ]]; then + copy_local_dir_to_server_directory LOCAL_ENFORCER_DIR + fi + + if [[ -n "${LOCAL_CORE_ENFORCER_DIR}" && "${COPY_LOCAL_CORE_ENFORCER}" == "true" ]]; then + copy_local_dir_to_server_directory LOCAL_CORE_ENFORCER_DIR + fi + + if [[ -n "${LOCAL_DEPLOY_TOOL_DIR}" && "${COPY_LOCAL_DEPLOY_TOOL_DIR}" == "true" ]]; then + copy_local_dir_to_server_directory LOCAL_DEPLOY_TOOL_DIR + fi +} + +function copy_local_dir_to_server_directory { + local env_var=$1 + local src_dir=${!env_var} + local target_dir=$(basename $src_dir) + echo "Copying $env_var $src_dir into ./$target_dir" + if [[ -d $target_dir ]]; then + rm -rf $target_dir + fi + rsync -r $src_dir/* $target_dir --exclude node_modules --exclude build --exclude dist --exclude bin + tmp_dirs+=($target_dir) + export $env_var=$target_dir +} + +function build_docker_image { + if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then + build_cdn_enforcer_docker_image + else + build_enforcer_docker_image + fi +} + +function build_cdn_enforcer_docker_image { + container_name=${CONTAINER_NAME:-sample-${enforcer}-${cdn_container_type}} + container_tag=${CONTAINER_TAG:-latest} + + if [[ $cdn_container_type == "origin" ]]; then + build_cdn_origin + elif [[ $cdn_container_type == "test_endpoints" ]]; then + build_cdn_test_endpoints + elif [[ $cdn_container_type == "update_sample_site" ]]; then + echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" + build_cdn_update_sample_site + else + echo "No known docker file for CDN enforcer $enforcer: $cdn_container_type" + fi +} + +function build_cdn_origin { + echo "Building docker image $container_name:$container_tag" + docker build -t $container_name:$container_tag -f ./templates/origin/Dockerfile \ + --build-arg ENFORCER_NAME=$enforcer \ + --build-arg PORT=${PORT} \ + . +} + +function build_cdn_test_endpoints { + echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" + docker build -t $container_name:$container_tag -f ./servers/$enforcer/$cdn_container_type/Dockerfile \ + --build-arg ENFORCER_NAME=$enforcer \ + --build-arg PORT=${PORT} \ + --build-arg LOCAL_ENFORCER_DIR=${LOCAL_ENFORCER_DIR} \ + --build-arg LOCAL_CORE_ENFORCER_DIR=${LOCAL_CORE_ENFORCER_DIR} \ + --build-arg LOCAL_DEPLOY_TOOL_DIR=${LOCAL_DEPLOY_TOOL_DIR} \ + --build-arg PX_APP_ID=${PX_APP_ID} \ + --build-arg PX_AUTH_TOKEN=${PX_AUTH_TOKEN} \ + --build-arg PX_COOKIE_SECRET=${PX_COOKIE_SECRET} \ + . +} + +function build_cdn_update_sample_site { + echo "NOT YET READY" +} + +function build_enforcer_docker_image { + container_name=${CONTAINER_NAME:-sample-${enforcer}} + container_tag=${CONTAINER_TAG:-latest} + + echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" + + docker build -t $container_name:$container_tag \ + --build-arg PX_APP_ID=${PX_APP_ID} \ + --build-arg PX_AUTH_TOKEN=${PX_AUTH_TOKEN} \ + --build-arg PX_COOKIE_SECRET=${PX_COOKIE_SECRET} \ + --build-arg ENABLE_TEST_ENDPOINTS=${ENABLE_TEST_ENDPOINTS:-true} \ + --build-arg LOCAL_ENFORCER_DIR=${LOCAL_ENFORCER_DIR} \ + --build-arg LOCAL_CORE_ENFORCER_DIR=${LOCAL_CORE_ENFORCER_DIR} \ + --build-arg DOCKER_ENFORCER_DIR=${DOCKER_ENFORCER_DIR} \ + -f servers/$enforcer/Dockerfile . +} + +function remove_tmp_enforcer_directories { + for tmp_dir in "${tmp_dirs[@]}" + do + echo "Removing ./$tmp_dir" + rm -rf $tmp_dir + done +} + +main $@ \ No newline at end of file diff --git a/demo-site/scripts/clean_static_files.js b/demo-site/scripts/clean_static_files.js new file mode 100644 index 0000000..03becd4 --- /dev/null +++ b/demo-site/scripts/clean_static_files.js @@ -0,0 +1,58 @@ +//region imports +const fs = require('fs'); +const path = require('path'); +const process = require('process'); +const { forEachServer, getUserInput, capitalize } = require("../utils/utils"); +const { CONFIG_FILE_NAME } = require('../utils/constants'); +//endregion + +const FILE_TYPES_TO_DELETE = [".html", ".css"]; + +const main = async () => { + if (!(await confirmDeletion())) { + console.log("Public directories will not be deleted."); + return; + } + + forEachServer((serverName, serverPath, serverConfig) => { + cleanStaticFiles(serverPath, serverConfig); + console.log(`Cleaned static files for ${capitalize(serverName)}`); + }); +}; + +const confirmDeletion = async () => { + const FLAG_INDEX = 2; + const CONFIRM_DELETION_FLAG = "-y"; + const YES_ANSWERS = ["yes", "y"]; + + if (process.argv[FLAG_INDEX] === CONFIRM_DELETION_FLAG) { + return true; + } + + const response = await getUserInput("Are you sure you want to delete static files in all servers?"); + return YES_ANSWERS.includes(response.toLowerCase()); +}; + +const cleanStaticFiles = (serverPath, config) => { + if (!config.site_config || !config.site_config.public_output_dir) { + console.error(`No property "public_output_dir" in ${serverPath}/${CONFIG_FILE_NAME}.json: ${config}`); + process.exit(1); + } + + if (config.site_config.public_output_dir === ".") { + fs.readdirSync(serverPath).forEach((filename) => { + for (const filetype of FILE_TYPES_TO_DELETE) { + if (filename.endsWith(filetype)) { + fs.rmSync(path.join(serverPath, filename)); + } + } + }); + } else { + const publicFilesPath = path.join(serverPath, config.site_config.public_output_dir); + if (fs.existsSync(publicFilesPath)) { + fs.rmdirSync(publicFilesPath, { recursive: true, force: true }); + } + } +}; + +main(); \ No newline at end of file diff --git a/demo-site/scripts/common.sh b/demo-site/scripts/common.sh new file mode 100644 index 0000000..afb9578 --- /dev/null +++ b/demo-site/scripts/common.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +shared_config_json_file=./shared_config.json +cdn_enforcers="fastly:cloudflare:lambda:akamai:fastly_js" +tmp_dirs=() + +function is_cdn_enforcer { + echo "$(contains $cdn_enforcers $1)" +} + +function contains { + [[ ":$1:" =~ ":$2:" ]] && echo "true" || echo "false" +} + +function validate_and_set_args { + if [[ -z $1 ]]; then + echo "For which server?" + read enforcer + else + enforcer=$1 + fi + + if [[ ! -d "./servers/$enforcer" ]]; then + echo "No directory found for enforcer $enforcer, exiting..." + exit 1 + fi + + config_json_file=./servers/$enforcer/config.json + if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then + if [[ -z $2 || $2 != "origin" && $2 != "test_endpoints" && $2 != "update_sample_site" ]]; then + echo "origin, test_endpoints, or update_sample_site?" + read cdn_container_type + else + cdn_container_type=$2 + fi + + if [[ ! -d "./servers/$enforcer/$cdn_container_type" ]]; then + echo "No directory $cdn_container_type found for enforcer $enforcer, exiting..." + exit 1 + fi + fi +} + +function get_value_from_config { + desired_value=$1 + json_file=$2 + if [[ ! -f $json_file ]]; then + echo "" + return + fi + value=$(cat $json_file | grep $desired_value | cut -d ":" -f2 | tr -d '\ \," "') + echo $value +} + +function use_env_or_config_value { + env_var=$1 + config_key=$2 + required=$3 + if [[ -z ${!env_var} ]]; then + export $env_var="$(get_value_from_config $config_key $config_json_file)" + fi + + if [[ -z ${!env_var} ]]; then + export $env_var="$(get_value_from_config $config_key $shared_config_json_file)" + fi + + if [[ "$required" == "true" && -z ${!env_var} ]]; then + echo "Unable to initialize mandatory env variable ${env_var}!" + exit 1 + fi +} \ No newline at end of file diff --git a/demo-site/scripts/create_px_configs.js b/demo-site/scripts/create_px_configs.js new file mode 100644 index 0000000..f7f8469 --- /dev/null +++ b/demo-site/scripts/create_px_configs.js @@ -0,0 +1,22 @@ +// region imports +const path = require('path'); +const sharedConfig = require('../shared_config.json'); +const { forEachServer, sortObjectAlphabeticallyByKey, capitalize, saveJson } = require('../utils/utils'); +const { PX_ENFORCER_CONFIG_FILE_NAME } = require('../utils/constants'); +// endregion + +const main = () => { + forEachServer((serverName, serverPath, serverConfig) => { + const pxConfig = mergeConfigs(serverConfig, sharedConfig); + saveJson(path.join(serverPath, `${PX_ENFORCER_CONFIG_FILE_NAME}`), pxConfig); + console.log(`Successfully created ${PX_ENFORCER_CONFIG_FILE_NAME} for ${capitalize(serverName)}`); + }); +}; + +const mergeConfigs = (serverConfig, sharedConfig) => { + const { enforcer_credentials, enforcer_config_override } = serverConfig; + const { enforcer_config } = sharedConfig; + return sortObjectAlphabeticallyByKey(Object.assign({}, enforcer_config, enforcer_credentials, enforcer_config_override)); +}; + +main(); diff --git a/demo-site/scripts/create_static_files.js b/demo-site/scripts/create_static_files.js new file mode 100644 index 0000000..000ee92 --- /dev/null +++ b/demo-site/scripts/create_static_files.js @@ -0,0 +1,105 @@ +//region imports +const fs = require('fs'); +const path = require('path'); +const sharedConfig = require('../shared_config.json'); +const { forEachServer, normalizeEnforcerName } = require("../utils/utils"); +const { CONFIG_FILE_NAME } = require('../utils/constants'); +//endregion + +const TEMPLATE_INDICATOR = sharedConfig.site_config.template_indicator; +const REPLACE_VARIABLE_REGEX = /\${[A-Za-z1-9_]*}/gi; + +const main = () => { + const publicTemplateDir = path.join(__dirname, "..", sharedConfig.site_config.public_template_dir); + forEachServer((serverName, serverPath, serverConfig) => { + const publicDir = createPublicDirForServer(serverPath, serverConfig); + if (!publicDir) { + console.error(`Couldn't create public dir for ${serverName} server`); + process.exit(1); + } + + if (!copyStaticFiles(publicTemplateDir, serverConfig, serverName, publicDir)) { + console.error(`Could not copy static files for ${normalizeEnforcerName(serverName)}. Skipping...`); + } else { + console.log(`Successfully created static files for ${normalizeEnforcerName(serverName)}`); + } + }); +}; + +const copyStaticFiles = (publicTemplateDir, serverConfig, serverName, publicDir) => { + const replaceVariableMaps = createReplacementVariableMaps(serverConfig, serverName); + if (!replaceVariableMaps) { + return false; + } + + const fileNames = fs.readdirSync(publicTemplateDir); + for (const fileName of fileNames) { + copyFileToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); + } + return true; +}; + +const copyFileToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { + if (fileName.includes(TEMPLATE_INDICATOR)) { + copyTemplateToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); + } else { + copyStaticFileToServerDirectory(fileName, publicTemplateDir, publicDir); + } +}; + +const copyTemplateToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { + const template = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); + for (const { templateIndicatorReplacement, replacementMap } of replaceVariableMaps) { + const fileContents = fillInTemplate(template, replacementMap); + const newFileName = fileName.replace(TEMPLATE_INDICATOR, templateIndicatorReplacement); + fs.writeFileSync(path.join(publicDir, newFileName), fileContents); + } +}; + +const copyStaticFileToServerDirectory = (fileName, publicTemplateDir, publicDir) => { + const fileContents = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); + fs.writeFileSync(path.join(publicDir, fileName), fileContents); +}; + +const fillInTemplate = (template, replaceVariableMap) => { + return template.replace(REPLACE_VARIABLE_REGEX, (matched) => replaceVariableMap[matched]); +} + +const createReplacementVariableMaps = (config, enforcerName) => { + const appId = config.enforcer_credentials.px_app_id; + if (appId == null || appId.length === 0) { + console.error(`No px_app_id found in ${enforcerName}/${CONFIG_FILE_NAME}!`); + return null; + } + const appIdSubstr = appId.substr(2); + return [ + createReplacementInfo("", appId, `//client.px-cloud.net/${appId}/main.min.js`, enforcerName), + createReplacementInfo(".firstparty", appId, `/${appIdSubstr}/init.js`, enforcerName) + ]; +}; + +const createReplacementInfo = (templateIndicatorReplacement, appId, sensorSrcUrl, enforcerName) => { + return { + templateIndicatorReplacement, + replacementMap: { + "${app_id}": appId, + "${sensor_src_url}": sensorSrcUrl, + "${enforcer_name}": normalizeEnforcerName(enforcerName) + } + }; +}; + +const createPublicDirForServer = (serverPath, config) => { + if (!config.site_config || !config.site_config.public_output_dir) { + console.error(`No property "public_output_dir" in ${serverPath}/${CONFIG_FILE_NAME}: ${config}`); + return null; + } + + const publicFilesPath = path.join(serverPath, config.site_config.public_output_dir); + if (!fs.existsSync(publicFilesPath)) { + fs.mkdirSync(publicFilesPath); + } + return publicFilesPath; +}; + +main(); diff --git a/demo-site/scripts/run_docker.sh b/demo-site/scripts/run_docker.sh new file mode 100755 index 0000000..5f9ac80 --- /dev/null +++ b/demo-site/scripts/run_docker.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +function main { + source ./scripts/common.sh + + validate_and_set_args $@ + + set_variables $@ + + run_docker_container +} + +function set_variables { + if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then + container_name=${CONTAINER_NAME:-sample-${enforcer}-${cdn_container_type}} + else + container_name=${CONTAINER_NAME:-sample-${enforcer}} + fi + + container_tag=${CONTAINER_TAG:-latest} + port=${PORT:-3000} + + if [[ "$2" == "mount" ]]; then + should_mount=true + fi +} + +function run_docker_container { + echo "Running $container_name:$container_tag" + if [[ "$should_mount" == "true" ]]; then + mount_and_run_docker + else + run_docker + fi +} + +function mount_and_run_docker { + use_env_or_config_value LOCAL_ENFORCER_DIR local_enforcer_dir true + use_env_or_config_value DOCKER_ENFORCER_DIR docker_enforcer_dir true + use_env_or_config_value LOCAL_CORE_ENFORCER_DIR local_core_enforcer_dir false + use_env_or_config_value DOCKER_CORE_ENFORCER_DIR docker_core_enforcer_dir false + + if [[ -n "$LOCAL_CORE_ENFORCER_DIR" || -n "$DOCKER_CORE_ENFORCER_DIR" ]]; then + echo "mounting local enforcer dir ${LOCAL_ENFORCER_DIR} and local core enforcer dir ${LOCAL_CORE_ENFORCER_DIR}" + docker run -it -p $port:$port \ + -v ${LOCAL_ENFORCER_DIR}:${DOCKER_ENFORCER_DIR} \ + -v ${LOCAL_CORE_ENFORCER_DIR}:${DOCKER_CORE_ENFORCER_DIR} \ + $container_name:$container_tag + else + echo "mounting local enforcer dir ${LOCAL_ENFORCER_DIR}" + docker run -it -p $port:$port \ + -v ${LOCAL_ENFORCER_DIR}:${DOCKER_ENFORCER_DIR} \ + $container_name:$container_tag + fi +} + +function run_docker { + docker run -it -p $port:$port $container_name:$container_tag +} + +main $@ \ No newline at end of file diff --git a/demo-site/servers/nodejs/app.js b/demo-site/servers/nodejs/app.js new file mode 100644 index 0000000..ee36eab --- /dev/null +++ b/demo-site/servers/nodejs/app.js @@ -0,0 +1,202 @@ +const path = require('path'); +const axios = require('axios'); +const express = require('express'); +const cookieParser = require('cookie-parser'); +const formData = require("express-form-data"); +const perimeterx = require('perimeterx-node-express'); + +const pxConfigJson = require('./px_config.json'); + +const PORT = 3000; + +var PxMiddleware; +var PxCdMiddleware; +let pxConfig; +var PxCdInterval; +var pxInstance; + +const main = () => { + pxConfig = initializeConfigs(); + + const app = initializeApp(); + if (process.env.ENABLE_TEST_ENDPOINTS === "true") { + setAdditionalActivityHandler(pxConfig); + setCustomParam(pxConfig); + } + setPxMiddleware(app); + setRoutes(app); + setTestEndpoints(app); + setStaticRoutes(app); + + const server = app.listen(PORT, '0.0.0.0', function () { + console.log(`NodeJS sample site is listening on port ${PORT}!`) + }); + + process.on('SIGINT', () => { + console.log('Closing http server...'); + server.close(); + process.exit(0); + }); +} + +const initializeApp = () => { + const app = express(); + app.use(cookieParser()); + app.use(express.json()); + app.use(express.urlencoded()); + // support form-data/multipart bodies + app.use(formData.parse()); + app.use(formData.format()); + app.use(formData.stream()); + app.use(formData.union()); + app.use((req, res, next) => { + console.log(req.method, req.path); + next(); + }) + return app; +} + +const setPxMiddleware = (app) => { + pxInstance = perimeterx.new(pxConfig) + PxMiddleware = pxInstance.middleware; + app.use(PxMiddlewareWrap); + + if (pxInstance.cdEnforcer) { + PxCdMiddleware = pxInstance.cdMiddleware; + PxCdInterval = pxInstance.cdEnforcer.setIntervalId; + app.use(PxCdMiddleware); + } + app.use((req, res, next) => { + for (const [name, value] of Object.entries(req.headers)) { + res.setHeader(name, value); + } + next(); + }); +} + +const initializeConfigs = () => { + return addEnvConfigs(pxConfigJson); +} + +const setAdditionalActivityHandler = (pxConfig) => { + pxConfig['px_additional_activity_handler'] = (pxCtx) => { + const { uri, pxde, pxdeVerified, score } = pxCtx; + axios.post(pxConfig.px_backend_url + "/additional" + uri, { + _pxde: pxde, + pxdeVerified: pxdeVerified, + pxScore: score + }).catch((e) => console.log(e.message)); + }; +} + +const setCustomParam = (pxConfig) => { + pxConfig['px_enrich_custom_parameters'] = (px_context, px_config)=>{ + let customParams = []; + for (let i = 1; i < 3; i++) { + let param_key = `custom_param${i}`; + let value = `test${i}`; + customParams[param_key] = value; + } + for (let i = 3; i < 7; i++) { + let param_key = `custom_param${i}`; + let value = i; + customParams[param_key] = value; + } + for (let i = 7; i <= 12; i++) { + let param_key = `custom_param${i}`; + let value = null; + customParams[param_key] = value; + } + return customParams; + }; +} + +const setTestEndpoints = (app) => { + app.post('/config', function (req, res, next) { + if(process.env.ENABLE_TEST_ENDPOINTS === 'false'){ + return res.sendStatus(404); + } + let newConfig = req.body; + // merge new config into pxConfig + Object.assign(pxConfig, newConfig) + if (pxConfig['px_csp_enabled']) { + clearInterval(PxCdInterval); + } + var pxInstance = perimeterx.new(pxConfig); + PxMiddleware = pxInstance.middleware; + setAdditionalActivityHandler(pxConfig); + setCustomParam(pxConfig) + res.sendStatus(200); + }); + + app.get('/supported-features', function (req, res, next) { + if(process.env.ENABLE_TEST_ENDPOINTS === 'false'){ + return res.sendStatus(404); + } + const supportedFeatures = require('perimeterx-node-express/px_metadata.json'); + return res.json(supportedFeatures); + }); + + app.get('/test-app-credentials', function (req, res){ + if (process.env.ENABLE_TEST_ENDPOINTS === 'false'){ + return res.sendStatus(404); + } + const test_app_credentials = { + "px_app_id": pxConfig.px_app_id, + "px_cookie_secret": pxConfig.px_cookie_secret + }; + return res.json(test_app_credentials) + }); +} + +const setRoutes = (app) => { + app.get('/', function (req, res) { + res.sendFile(__dirname + '/public/index' + getRequiredSuffix() + '.html'); + }) + + app.post('/login', function (req, res, next) { + const loginSuccessful = req.body.username === 'pxUser' && req.body.password === '1234'; + res.pxLoginSuccessful = loginSuccessful; + if (loginSuccessful) { + res.sendFile(__dirname + '/public/profile' + getRequiredSuffix() + '.html'); + } else { + res.status(301).redirect('/'); + } + }); + + app.get('/logout', function (req, res) { + res.redirect('/'); + }); + +} + +const setStaticRoutes = (app) => { + app.use(express.static(path.join(__dirname, 'public'))); +} + +const addEnvConfigs = (config) => { + const envConfigs = { + "px_app_id" : process.env.PX_APP_ID, + "px_cookie_secret" : process.env.PX_COOKIE_SECRET, + } + for (const key in envConfigs){ + if (!envConfigs[key] || config[key] !== ""){ + delete envConfigs[key]; + } + } + Object.assign(config, envConfigs); + return config; +}; + +const getRequiredSuffix = () => { + return pxConfig.px_first_party_enabled ? ".firstparty" : ""; +}; + +const PxMiddlewareWrap = (req, res, next) => { + if (pxConfig.px_filter_by_route && pxConfig.px_filter_by_route.includes(req.path)){ + return next(); + } + PxMiddleware(req, res, next); +}; + +main(); \ No newline at end of file diff --git a/demo-site/servers/nodejs/config.inc.json b/demo-site/servers/nodejs/config.inc.json new file mode 100644 index 0000000..8f10fe7 --- /dev/null +++ b/demo-site/servers/nodejs/config.inc.json @@ -0,0 +1,16 @@ +{ + "site_config": { + "public_output_dir": "public", + "local_enforcer_dir": "", + "local_core_enforcer_dir": "", + "port": 3000 + }, + "enforcer_credentials": { + "px_app_id": "PXZ7wl3dvS", + "px_auth_token": "", + "px_cookie_secret": "" + }, + "enforcer_config_override": { + + } +} \ No newline at end of file diff --git a/demo-site/servers/nodejs/config.json b/demo-site/servers/nodejs/config.json new file mode 100644 index 0000000..e8e0cd8 --- /dev/null +++ b/demo-site/servers/nodejs/config.json @@ -0,0 +1,23 @@ +{ + "site_config": { + "public_output_dir": "public", + "local_enforcer_dir": "../../", + "local_core_enforcer_dir": "/Users/danbezalel/workspace/enforcers/perimeterx-node-core", + "port": 3000 + }, + "enforcer_credentials": { + "px_app_id": "PXnEpdw6lS", + "px_auth_token": "${{ secrets.TEST_AUTH_TOKEN }}", + "px_cookie_secret": "" + }, + "enforcer_config_override": { + "px_blocking_score": 70, + "px_module_mode": "active_blocking", + "px_s2s_timeout": 1000, + "px_user_agent_max_length": 8528, + "px_risk_cookie_max_length": 2048, + "px_risk_cookie_max_iterations": 5000, + "px_risk_cookie_min_iterations": 1, + "px_custom_cookie_header": "x-px-cookies" + } +} \ No newline at end of file diff --git a/demo-site/servers/nodejs/package.json b/demo-site/servers/nodejs/package.json new file mode 100644 index 0000000..b39c2da --- /dev/null +++ b/demo-site/servers/nodejs/package.json @@ -0,0 +1,27 @@ +{ + "name": "nodejs", + "version": "1.0.0", + "description": "", + "main": "app.js", + "scripts": { + "start": "node app.js", + "dev": "nodemon app.js", + "test": "echo \"Error: no test specified\" && exit 1", + "cover": "nyc node app.js" + }, + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.21.1", + "body-parser": "^1.19.0", + "cookie-parser": "^1.4.3", + "express": "^4.14.0", + "express-form-data": "^2.0.17", + "perimeterx-node-core": "^3.11.0" + }, + "devDependencies": { + "mocha": "^5.2.0", + "nodemon": "^2.0.7", + "nyc": "^13.2.0" + } +} diff --git a/demo-site/servers/nodejs/px_config.json b/demo-site/servers/nodejs/px_config.json new file mode 100644 index 0000000..d8b6ff8 --- /dev/null +++ b/demo-site/servers/nodejs/px_config.json @@ -0,0 +1,87 @@ +{ +"px_advanced_blocking_response_enabled": true, +"px_app_id": "", +"px_auth_token": "", +"px_blocking_score": 70, +"px_bypass_monitor_header": "", +"px_cookie_secret": "", +"px_css_ref": "style.css", +"px_custom_block_page_url": "", +"px_custom_cookie_header": "x-px-cookies", +"px_custom_logo": "https://storage.googleapis.com/perimeterx-logos/primary_logo_red_cropped.png", +"px_enforced_routes": [], +"px_filter_by_extension": [ +".css", +".bmp", +".tif", +".ttf", +".docx", +".woff2", +".js", +".pict", +".tiff", +".eot", +".xlsx", +".jpg", +".csv", +".eps", +".woff", +".xls", +".jpeg", +".doc", +".ejs", +".otf", +".pptx", +".gif", +".pdf", +".swf", +".svg", +".ps", +".ico", +".pls", +".midi", +".svgz", +".class", +".png", +".ppt", +".mid", +".webp", +".jar", +".json", +".xml" +], +"px_filter_by_route": [ +"/supported-features", +"/test-app-credentials", +"/config" +], +"px_first_party_enabled": true, +"px_ip_headers": [], +"px_js_ref": "index.js", +"px_logger_severity": "debug", +"px_login_credentials_extraction": [ +{ +"path": "/login", +"method": "post", +"sent_through": "body", +"user_field": "username", +"pass_field": "password" +} +], +"px_login_credentials_extraction_enabled": true, +"px_max_activity_batch_size": 1, +"px_max_buffer_len": 1, +"px_module_enabled": true, +"px_module_mode": "active_blocking", +"px_proxy_url": "", +"px_risk_cookie_max_iterations": 5000, +"px_risk_cookie_max_length": 2048, +"px_risk_cookie_min_iterations": 1, +"px_s2s_timeout": 1000, +"px_sensitive_headers": [ +"cookie", +"cookies" +], +"px_sensitive_routes": [], +"px_user_agent_max_length": 8528 +} \ No newline at end of file diff --git a/demo-site/shared_config.json b/demo-site/shared_config.json new file mode 100644 index 0000000..3e1bbf4 --- /dev/null +++ b/demo-site/shared_config.json @@ -0,0 +1,83 @@ +{ + "site_config": { + "public_template_dir": "./templates/static_files", + "template_indicator": ".template" + }, + "enforcer_config": { + "px_module_enabled": true, + "px_first_party_enabled": true, + "px_blocking_score": 100, + "px_module_mode": "active_blocking", + "px_sensitive_headers": ["cookie", "cookies"], + "px_sensitive_routes": [], + "px_enforced_routes": [], + "px_ip_headers": [], + "px_custom_logo": "https://storage.googleapis.com/perimeterx-logos/primary_logo_red_cropped.png", + "px_css_ref": "style.css", + "px_js_ref": "index.js", + "px_logger_severity": "debug", + "px_s2s_timeout": 1000, + "px_filter_by_route": [ + "/supported-features", + "/test-app-credentials", + "/config" + ], + "px_advanced_blocking_response_enabled": true, + "px_proxy_url": "", + "px_max_activity_batch_size": 1, + "px_max_buffer_len": 1, + "px_bypass_monitor_header": "", + "px_custom_block_page_url": "", + "px_filter_by_extension": [ + ".css", + ".bmp", + ".tif", + ".ttf", + ".docx", + ".woff2", + ".js", + ".pict", + ".tiff", + ".eot", + ".xlsx", + ".jpg", + ".csv", + ".eps", + ".woff", + ".xls", + ".jpeg", + ".doc", + ".ejs", + ".otf", + ".pptx", + ".gif", + ".pdf", + ".swf", + ".svg", + ".ps", + ".ico", + ".pls", + ".midi", + ".svgz", + ".class", + ".png", + ".ppt", + ".mid", + ".webp", + ".jar", + ".json", + ".xml" + ], + "px_user_agent_max_length": 8528, + "px_risk_cookie_max_length": 2048, + "px_risk_cookie_max_iterations": 5000, + "px_login_credentials_extraction_enabled": true, + "px_login_credentials_extraction": [{ + "path": "/login", + "method": "post", + "sent_through": "body", + "user_field": "username", + "pass_field": "password" + }] + } +} \ No newline at end of file diff --git a/demo-site/templates/origin/Dockerfile b/demo-site/templates/origin/Dockerfile new file mode 100644 index 0000000..9580996 --- /dev/null +++ b/demo-site/templates/origin/Dockerfile @@ -0,0 +1,39 @@ +FROM node:16-alpine as builder +ARG ENFORCER_NAME + +WORKDIR /tmp/sample-sites + +COPY shared_config.json . +COPY scripts scripts +COPY utils utils +COPY templates/static_files templates/static_files +COPY servers/${ENFORCER_NAME} servers/${ENFORCER_NAME} + +RUN node scripts/create_static_files.js && node scripts/create_px_configs.js + +FROM node:16-alpine as origin +ARG ENFORCER_NAME + +WORKDIR /app +COPY utils utils + +WORKDIR /app/templates/origin +COPY templates/origin/package.json . +RUN npm install +COPY templates/origin/*.js . + +WORKDIR /app/servers/${ENFORCER_NAME} +COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/px_config.json . +COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/config.inc.json . + +WORKDIR /app/servers/${ENFORCER_NAME}/origin +COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/origin/public/ public/ +COPY servers/${ENFORCER_NAME}/origin/package.json . +RUN npm install +COPY servers/${ENFORCER_NAME}/origin/*.js . + +ARG PORT=3000 +ENV PORT ${PORT} + +EXPOSE ${PORT} +CMD ["npm", "start"] \ No newline at end of file diff --git a/demo-site/templates/origin/origin_app.js b/demo-site/templates/origin/origin_app.js new file mode 100644 index 0000000..db5553c --- /dev/null +++ b/demo-site/templates/origin/origin_app.js @@ -0,0 +1,73 @@ +const path = require('path'); +const { env } = require('process'); +const express = require('express'); +const cookieParser = require('cookie-parser'); +const { + INDEX_ROUTE, + LOGIN_ROUTE, + LOGOUT_ROUTE, + EXPECTED_USERNAME, + EXPECTED_PASSWORD, + FIRST_PARTY_STATIC_FILE_SUFFIX, + THIRD_PARTY_STATIC_FILE_SUFFIX +} = require("../../utils/constants"); + +class OriginApp { + constructor(enforcerType) { + this.enforcerType = enforcerType; + this.app = this.createApp(); + } + + createApp() { + const app = express(); + app.use(cookieParser()); + app.use(express.json()); + app.use(express.urlencoded({ extended: true })); + app.use((req, res, next) => { + console.log(`${req.method} ${req.path}`); + res.set(req.headers); + next(); + }) + return app; + } + + SetRoutes(staticRoutesDir, isFirstParty) { + this.app.use(express.static(staticRoutesDir, {index: false})); + + const htmlSuffix = isFirstParty ? + FIRST_PARTY_STATIC_FILE_SUFFIX : THIRD_PARTY_STATIC_FILE_SUFFIX; + + this.app.get(INDEX_ROUTE, (req, res) => { + const htmlPageName = `index${htmlSuffix}.html`; + res.sendFile(path.join(staticRoutesDir, htmlPageName)); + }); + + this.app.post(LOGIN_ROUTE, (req, res) => { + if (req.body.username === EXPECTED_USERNAME && + req.body.password === EXPECTED_PASSWORD) { + res.sendFile(path.join(staticRoutesDir, `profile${htmlSuffix}.html`)); + } else { + res.redirect(INDEX_ROUTE); + } + }); + + this.app.get(LOGOUT_ROUTE, (req, res) => { + res.redirect(INDEX_ROUTE); + }); + } + + Start() { + const port = env.PORT || 3000; + const server = this.app.listen(port, '0.0.0.0', () => { + console.log(`${this.enforcerType} origin listening on port ${port}!`) + }); + + process.on('SIGINT', () => { + console.log('\nClosing http server...'); + server.close(); + process.exit(0); + }); + } +} + +module.exports = OriginApp; diff --git a/demo-site/templates/origin/package.json b/demo-site/templates/origin/package.json new file mode 100644 index 0000000..4c19da8 --- /dev/null +++ b/demo-site/templates/origin/package.json @@ -0,0 +1,12 @@ +{ + "name": "origin-utils", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "cookie-parser": "^1.4.5", + "express": "^4.17.1" + } +} diff --git a/demo-site/templates/static_files/index.template.html b/demo-site/templates/static_files/index.template.html new file mode 100644 index 0000000..9b57590 --- /dev/null +++ b/demo-site/templates/static_files/index.template.html @@ -0,0 +1,44 @@ + + + + PerimeterX ${enforcer_name} SDK Sample + + + + + +
+
+
+ +
+
+
+ + + \ No newline at end of file diff --git a/demo-site/templates/static_files/profile.template.html b/demo-site/templates/static_files/profile.template.html new file mode 100644 index 0000000..7eb0efb --- /dev/null +++ b/demo-site/templates/static_files/profile.template.html @@ -0,0 +1,39 @@ + + + + PerimeterX Login Successful + + + + + +
+
+

Hello, pxUser


+ +
+ +
+ + +
+ +
+ + \ No newline at end of file diff --git a/demo-site/templates/static_files/style.css b/demo-site/templates/static_files/style.css new file mode 100644 index 0000000..e218f83 --- /dev/null +++ b/demo-site/templates/static_files/style.css @@ -0,0 +1,115 @@ +body { + background-color: #848484; +} + +.container { + margin-top:150px; +} + +.box { + margin: 0 auto; + padding-top:20px; + text-align: center; + color: white; + width:350px; + background-color: black; + border-radius: 12px; + height: 350px; + box-shadow: 3px 3px 3px #ED1C24; +} + +.login-box { + margin: 0 auto; + padding-top:20px; + text-align: center; + color: white; + background-color: black; + border-radius: 12px; + box-shadow: 3px 3px 3px #ED1C24; +} + +#logout { + margin-top:120px; +} + +#logout a { + color:black; + text-decoration: none; +} + +.form-signin +{ + max-width: 330px; + padding: 15px; + margin: 0 auto; +} +.form-signin .form-signin-heading, .form-signin .checkbox +{ + margin-bottom: 10px; +} +.form-signin .checkbox +{ + font-weight: normal; +} +.form-signin .form-control +{ + position: relative; + font-size: 16px; + height: auto; + padding: 10px; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.form-signin .form-control:focus +{ + z-index: 2; +} +.form-signin input[type="text"] +{ + margin-bottom: -1px; + border-bottom-left-radius: 0; + border-bottom-right-radius: 0; +} +.form-signin input[type="password"] +{ + margin-bottom: 10px; + border-top-left-radius: 0; + border-top-right-radius: 0; +} +.account-wall +{ + margin-top: 20px; + padding: 40px 0px 20px 0px; + background-color: black; + -moz-box-shadow: 0px 2px 2px rgba(0, 0, 0, 0.3); + -webkit-box-shadow: 0px 2px 2px rgba(0, 0, 0, 0.3); + box-shadow: 0px 2px 2px rgba(0, 0, 0, 0.3); + border-radius: 12px; +} +.login-title +{ + color: white; + font-size: 18px; + font-weight: 400; + display: block; +} +.profile-img +{ + width: 96px; + height: 96px; + margin: 0 auto 10px; + display: block; + -moz-border-radius: 50%; + -webkit-border-radius: 50%; + border-radius: 50%; +} +.need-help +{ + margin-top: 10px; +} +.new-account +{ + display: block; + margin-top: 10px; +} \ No newline at end of file diff --git a/demo-site/templates/test_endpoints/package.json b/demo-site/templates/test_endpoints/package.json new file mode 100644 index 0000000..9d4988e --- /dev/null +++ b/demo-site/templates/test_endpoints/package.json @@ -0,0 +1,12 @@ +{ + "name": "test-endpoints-utils", + "version": "1.0.0", + "description": "", + "author": "", + "license": "ISC", + "dependencies": { + "cookie-parser": "^1.4.3", + "express": "^4.14.0" + }, + "devDependencies": {} +} diff --git a/demo-site/templates/test_endpoints/test_endpoints_app.js b/demo-site/templates/test_endpoints/test_endpoints_app.js new file mode 100644 index 0000000..a0bfee3 --- /dev/null +++ b/demo-site/templates/test_endpoints/test_endpoints_app.js @@ -0,0 +1,94 @@ +const path = require('path'); +const express = require('express'); +const cookieParser = require('cookie-parser'); +const { env } = require('process'); +const { loadJson } = require("../../utils/utils"); +const { + TEST_APP_CREDENTIALS_ENDPOINT, + SUPPORTED_FEATURES_ENDPOINT, + CONFIG_ENDPOINT, + PX_METADATA_FILE_NAME +} = require('../../utils/constants'); + +class TestEndpointsApp { + constructor(enforcerType) { + this.enforcerType = enforcerType; + this.app = this.createApp(); + } + + createApp() { + const app = express(); + app.use(cookieParser()); + app.use(express.json()); + app.use(express.urlencoded({ + extended: true + })); + return app; + } + + SetTestAppCredentialsEndpoint({ px_app_id, px_cookie_secret }) { + this.app.get(TEST_APP_CREDENTIALS_ENDPOINT, (req, res) => { + console.log(`GET ${TEST_APP_CREDENTIALS_ENDPOINT}`); + const credentials = { + px_app_id, + px_cookie_secret + }; + res.json(credentials); + }); + } + + SetSupportedFeaturesEndpoint(enforcerDir) { + this.app.get(SUPPORTED_FEATURES_ENDPOINT, (req, res) => { + console.log(`GET ${SUPPORTED_FEATURES_ENDPOINT}`); + try { + const pxMetadata = loadJson(path.join(enforcerDir, PX_METADATA_FILE_NAME)); + res.json(pxMetadata); + } catch (err) { + console.error(err); + res.sendStatus(500); + } + }); + } + + /** + * @param changeEnforcerConfigCallback: function (enforcerConfig, serverConfig) => boolean + * the return value indicates whether the enforcer config was successfuly updated + * + * @param serverConfig - object with structure identical to config.inc.json + * fields are filled in with either the JSON values or env variables + */ + SetConfigEndpoint(changeEnforcerConfigCallback, serverConfig) { + this.app.post(CONFIG_ENDPOINT, async (req, res) => { + console.log(`POST ${CONFIG_ENDPOINT}`); + try { + if (!changeEnforcerConfigCallback) { + console.log("Skipping changing enforcer config..."); + return res.sendStatus(200); + } + if (!(await changeEnforcerConfigCallback(req.body, serverConfig))) { + console.error("Unable to change enforcer config!"); + return res.sendStatus(500); + } + res.sendStatus(200); + } catch (err) { + console.error(err); + res.sendStatus(500); + } + }); + } + + Start() { + const port = env.PORT || 3000; + const server = this.app.listen(port, '0.0.0.0', () => { + console.log(`${this.enforcerType} test endpoints listening on port ${port}!`); + }); + + process.on('SIGINT', () => { + console.log('\nClosing http server...'); + server.close(); + process.exit(0); + }); + } +} + +module.exports = TestEndpointsApp; \ No newline at end of file diff --git a/demo-site/utils/cdn_deploy_tool_utils.js b/demo-site/utils/cdn_deploy_tool_utils.js new file mode 100644 index 0000000..d50ca5e --- /dev/null +++ b/demo-site/utils/cdn_deploy_tool_utils.js @@ -0,0 +1,82 @@ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); +const { loadJson, saveJson } = require("./utils"); +const { + CDN_DEPLOY_TOOL_RUN_CLI_COMMAND, + CDN_DEPLOY_TOOL_CONFIG_FILE_NAME, + CDN_DEPLOY_TOOL_BUILD_COMMAND +} = require("./constants"); + +const setCDNDeployToolPipeline = (deployToolPath, cdnType, deployType) => { + const deployToolConfigPath = path.join(deployToolPath, "configs", CDN_DEPLOY_TOOL_CONFIG_FILE_NAME); + const deployToolConfig = loadJson(deployToolConfigPath); + if (!deployToolConfig) { + console.error(`Cannot find ${deployToolConfigPath}!`); + return false; + } + + deployToolConfig.cdnType = cdnType; + deployToolConfig.deployType = deployType; + + saveJson(deployToolConfigPath, deployToolConfig); + return true; +}; + +const getCDNDeployToolPipelineConfig = (deployToolPath, pipelineName) => { + const pipelineConfigIncPath = path.join(deployToolPath, `configs/pipelines/${pipelineName}/pipelineConfig.inc.json`); + const pipelineConfigIncJson = loadJson(pipelineConfigIncPath); + if (pipelineConfigIncJson) { + return pipelineConfigIncJson; + } + console.error(`Cannot find ${pipelineConfigIncPath}!`); + return null; +}; + +const setCDNDeployToolPipelineConfig = (deployToolPath, pipelineName, pipelineConfig) => { + const pipelineConfigPath = path.join(deployToolPath, `configs/pipelines/${pipelineName}/pipelineConfig.json`); + saveJson(pipelineConfigPath, pipelineConfig); + return true; +} + +const runCDNDeployTool = (deployToolPath) => { + if (!isCDNDeployToolReady(deployToolPath)) { + return false; + } + console.log("\nRunning deploy pipeline"); + if (!runCDNDeployToolCommand(deployToolPath, CDN_DEPLOY_TOOL_RUN_CLI_COMMAND)) { + return false; + } + console.log("Deploy completed successfully!"); + return true; +} + +const isCDNDeployToolReady = (deployToolPath) => { + if (!fs.existsSync(path.join(deployToolPath, "node_modules"))) { + console.error("CDN Deploy tool dependencies are not installed"); + return false; + } + if (!fs.existsSync(path.join(deployToolPath, "build"))) { + console.info("CDN Deploy tool has not been compiled"); + if (!runCDNDeployToolCommand(deployToolPath, CDN_DEPLOY_TOOL_BUILD_COMMAND)) { + return false; + } + } + return true; +} + +const runCDNDeployToolCommand = (deployToolPath, command) => { + const output = execSync(command, { cwd: deployToolPath }); + console.log(output + '\n'); + let outputString = output.toString(); + console.log(outputString); + outputString = outputString.toLowerCase(); + return outputString.indexOf("error") === -1 && outputString.indexOf("fail") === -1; +} + +module.exports = { + setCDNDeployToolPipeline, + getCDNDeployToolPipelineConfig, + setCDNDeployToolPipelineConfig, + runCDNDeployTool +} \ No newline at end of file diff --git a/demo-site/utils/constants.js b/demo-site/utils/constants.js new file mode 100644 index 0000000..66bbf24 --- /dev/null +++ b/demo-site/utils/constants.js @@ -0,0 +1,55 @@ +// region imports +const path = require('path'); +// endregion + +const JSON_SPACING = 4; + +const SERVER_CONFIG_FILE_NAME = 'config.json'; +const SERVER_CONFIG_INC_FILE_NAME = "config.inc.json"; +const PX_ENFORCER_CONFIG_FILE_NAME = 'px_config.json'; + +const SERVERS_DIRECTORY_NAME = "servers"; +const SERVERS_DIRECTORY_PATH = path.join(__dirname, `../${SERVERS_DIRECTORY_NAME}`); + +const FIRST_PARTY_STATIC_FILE_SUFFIX = ".firstparty"; +const THIRD_PARTY_STATIC_FILE_SUFFIX = ""; + +const CDN_DEPLOY_TOOL_CONFIG_FILE_NAME = "cliConfig.json"; +const CDN_DEPLOY_TOOL_RUN_CLI_COMMAND = "npm run cli"; +const CDN_DEPLOY_TOOL_BUILD_COMMAND = "npm run build"; + +const PX_METADATA_FILE_NAME = "px_metadata.json"; + +const TEST_APP_CREDENTIALS_ENDPOINT = "/test-app-credentials"; +const SUPPORTED_FEATURES_ENDPOINT = "/supported-features"; +const CONFIG_ENDPOINT = "/config"; + +const INDEX_ROUTE = "/"; +const LOGIN_ROUTE = "/login"; +const LOGOUT_ROUTE = "/logout"; + +const EXPECTED_USERNAME = "pxUser"; +const EXPECTED_PASSWORD = "1234"; + +module.exports = { + JSON_SPACING, + SERVER_CONFIG_FILE_NAME, + SERVER_CONFIG_INC_FILE_NAME, + PX_ENFORCER_CONFIG_FILE_NAME, + SERVERS_DIRECTORY_NAME, + SERVERS_DIRECTORY_PATH, + FIRST_PARTY_STATIC_FILE_SUFFIX, + THIRD_PARTY_STATIC_FILE_SUFFIX, + CDN_DEPLOY_TOOL_CONFIG_FILE_NAME, + CDN_DEPLOY_TOOL_RUN_CLI_COMMAND, + CDN_DEPLOY_TOOL_BUILD_COMMAND, + PX_METADATA_FILE_NAME, + TEST_APP_CREDENTIALS_ENDPOINT, + SUPPORTED_FEATURES_ENDPOINT, + CONFIG_ENDPOINT, + INDEX_ROUTE, + LOGIN_ROUTE, + LOGOUT_ROUTE, + EXPECTED_USERNAME, + EXPECTED_PASSWORD +}; \ No newline at end of file diff --git a/demo-site/utils/utils.js b/demo-site/utils/utils.js new file mode 100644 index 0000000..7e05ed1 --- /dev/null +++ b/demo-site/utils/utils.js @@ -0,0 +1,115 @@ +// region imports +const fs = require('fs'); +const path = require('path'); +const process = require('process'); +const readline = require('readline'); +const { SERVERS_DIRECTORY_PATH, SERVER_CONFIG_FILE_NAME, SERVER_CONFIG_INC_FILE_NAME, JSON_SPACING } = require('./constants'); +// endregion + +const forEachServer = async (callback) => { + const serverDirectories = fs.readdirSync(SERVERS_DIRECTORY_PATH); + for (const serverName of serverDirectories) { + const serverPath = getServerAbsolutePath(serverName); + const serverConfig = getServerConfig(serverPath); + + if (!serverConfig) { + console.error("Couldn't get configs for", serverName); + continue; + } + + await callback(serverName, serverPath, serverConfig); + } +}; + +const getServerConfig = (serverPath) => { + if (!path.isAbsolute(serverPath)) { + serverPath = getServerAbsolutePath(serverPath); + } + const configJson = loadJson(path.join(serverPath, `${SERVER_CONFIG_FILE_NAME}`)); + if (configJson) { + return configJson; + } + + const configIncJson = loadJson(path.join(serverPath, `${SERVER_CONFIG_INC_FILE_NAME}`)); + if (configIncJson) { + console.log(`No ${serverPath}/${SERVER_CONFIG_FILE_NAME}, using ${SERVER_CONFIG_INC_FILE_NAME} instead!`); + return configIncJson; + } + + console.error(`No ${serverPath}/${SERVER_CONFIG_FILE_NAME} or ${SERVER_CONFIG_INC_FILE_NAME} files found!`); + return null; +}; + +const getServerAbsolutePath = (serverDir) => { + return path.join(SERVERS_DIRECTORY_PATH, serverDir); +}; + +const loadJson = (path) => { + if (fs.existsSync(path)) { + return JSON.parse(fs.readFileSync(path)); + } + return null; +} + +const saveJson = (filename, jsonObject) => { + fs.writeFileSync(filename, JSON.stringify(jsonObject, null, JSON_SPACING)); +} + +const sortObjectAlphabeticallyByKey = (object) => { + return Object.keys(object).sort().reduce((obj, key) => { + obj[key] = object[key]; + return obj; + }, {}); +}; + +const getUserInput = (query) => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + return new Promise((resolve) => rl.question(query + " ", (ans) => { + rl.close(); + resolve(ans); + })); +}; + +const capitalize = (string) => { + return string.charAt(0).toUpperCase() + string.slice(1); +}; + +const normalizeEnforcerName = (enforcerName) => { + return enforcerName.split(/[_\.\-]/g).map(capitalize).join(' '); +}; + +const allEnvVariablesExist = (requiredEnvVariables) => { + if (!requiredEnvVariables || !Array.isArray(requiredEnvVariables)) { + return false; + } + + const messages = []; + for (const variableName of requiredEnvVariables) { + if (!process.env[variableName]) { + messages.push(`You must define environment variable ${variableName}`); + } + } + + if (messages.length === 0) { + return true; + } + console.log(messages.join("\n")); + return false; +} + +module.exports = { + forEachServer, + getServerConfig, + getServerAbsolutePath, + loadJson, + saveJson, + sortObjectAlphabeticallyByKey, + getUserInput, + capitalize, + normalizeEnforcerName, + allEnvVariablesExist +}; \ No newline at end of file diff --git a/px_metadata.json b/px_metadata.json index dba065e..cf5bf4d 100644 --- a/px_metadata.json +++ b/px_metadata.json @@ -42,5 +42,22 @@ "sensitive_headers", "telemetry_command", "vid_extraction" + ], + "excluded_tests": [ + "test_additional_activity_handler_with_score_from_risk", + "test_additional_activity_handler_with_score_from_cookie", + "test_advanced_blocking_response", + "test_preflight_request_returns_custom_preflight_handler_response", + "test_pxde_extraction_s2s", + "test_pxde_extraction_unverified", + "test_pxde_extraction_verified", + "test_pxhd_should_return_on_set_cookie_header_when_received_from_risk", + "test_pxhd_should_be_on_set_cookie_even_if_domain_is_none", + "test_vid_extraction_on_first_party_xhr", + "test_block_activity_cookie_origin", + "test_page_requested_activity_cookie_origin", + "test_block_page_hard_block_response", + "test_risk_api_validate_cookie_origin", + "test_risk_cookie_valid_cookie_with_user_agent_bigger_than_max_length" ] } \ No newline at end of file From 2a92e69ea844eae40adff13e97d196955ecb1dbf Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Mon, 26 Jun 2023 21:37:05 +0300 Subject: [PATCH 02/13] ci for express --- demo-site/servers/nodejs/config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo-site/servers/nodejs/config.json b/demo-site/servers/nodejs/config.json index e8e0cd8..b093a9d 100644 --- a/demo-site/servers/nodejs/config.json +++ b/demo-site/servers/nodejs/config.json @@ -2,12 +2,12 @@ "site_config": { "public_output_dir": "public", "local_enforcer_dir": "../../", - "local_core_enforcer_dir": "/Users/danbezalel/workspace/enforcers/perimeterx-node-core", + "local_core_enforcer_dir": "", "port": 3000 }, "enforcer_credentials": { "px_app_id": "PXnEpdw6lS", - "px_auth_token": "${{ secrets.TEST_AUTH_TOKEN }}", + "px_auth_token": "", "px_cookie_secret": "" }, "enforcer_config_override": { From 23733dfce067b7346cb8d48d8ef568078de38a9e Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Tue, 27 Jun 2023 11:06:57 +0300 Subject: [PATCH 03/13] removed unnececery files --- demo-site/scripts/build_docker.sh | 161 ------------------ demo-site/scripts/common.sh | 72 -------- demo-site/scripts/run_docker.sh | 62 ------- demo-site/templates/origin/Dockerfile | 39 ----- demo-site/templates/origin/origin_app.js | 73 -------- demo-site/templates/origin/package.json | 12 -- .../templates/test_endpoints/package.json | 12 -- .../test_endpoints/test_endpoints_app.js | 94 ---------- demo-site/utils/cdn_deploy_tool_utils.js | 82 --------- 9 files changed, 607 deletions(-) delete mode 100755 demo-site/scripts/build_docker.sh delete mode 100644 demo-site/scripts/common.sh delete mode 100755 demo-site/scripts/run_docker.sh delete mode 100644 demo-site/templates/origin/Dockerfile delete mode 100644 demo-site/templates/origin/origin_app.js delete mode 100644 demo-site/templates/origin/package.json delete mode 100644 demo-site/templates/test_endpoints/package.json delete mode 100644 demo-site/templates/test_endpoints/test_endpoints_app.js delete mode 100644 demo-site/utils/cdn_deploy_tool_utils.js diff --git a/demo-site/scripts/build_docker.sh b/demo-site/scripts/build_docker.sh deleted file mode 100755 index 83d6782..0000000 --- a/demo-site/scripts/build_docker.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -set -e - -function main { - initialize $@ - - copy_local_directories - - build_docker_image - - remove_tmp_enforcer_directories -} - -function initialize { - source ./scripts/common.sh - - validate_and_set_args $@ - - initialize_environment_variables -} - -function is_deploy_tool_required { - if [[ "$(is_cdn_enforcer $enforcer)" == "true" && $cdn_container_type != "origin" ]]; then - echo "true" - else - echo "false" - fi -} - -function is_enforcer_required { - if [[ "$(is_cdn_enforcer $enforcer)" == "true" && $cdn_container_type == "origin" ]]; then - echo "false" - else - echo "true" - fi -} - -function initialize_environment_variables { - echo "Initializing environment variables" - use_env_or_config_value PX_APP_ID px_app_id true - use_env_or_config_value PX_AUTH_TOKEN px_auth_token true - use_env_or_config_value PX_COOKIE_SECRET px_cookie_secret true - use_env_or_config_value DOCKER_ENFORCER_DIR docker_enforcer_dir - - if [[ "$(is_enforcer_required)" == "true" ]]; then - use_env_or_config_value LOCAL_ENFORCER_DIR local_enforcer_dir true - use_env_or_config_value LOCAL_CORE_ENFORCER_DIR local_core_enforcer_dir - fi - - if [[ "$(is_deploy_tool_required)" == "true" ]]; then - use_env_or_config_value LOCAL_DEPLOY_TOOL_DIR local_deploy_tool_dir true - fi -} - -function copy_local_directories { - export COPY_LOCAL_ENFORCER=${COPY_LOCAL_ENFORCER:-true} - export COPY_LOCAL_CORE_ENFORCER=${COPY_LOCAL_CORE_ENFORCER:-true} - export COPY_LOCAL_DEPLOY_TOOL_DIR=${COPY_LOCAL_DEPLOY_TOOL_DIR:-true} - - if [[ -n "${LOCAL_ENFORCER_DIR}" && "${COPY_LOCAL_ENFORCER}" == "true" ]]; then - copy_local_dir_to_server_directory LOCAL_ENFORCER_DIR - fi - - if [[ -n "${LOCAL_CORE_ENFORCER_DIR}" && "${COPY_LOCAL_CORE_ENFORCER}" == "true" ]]; then - copy_local_dir_to_server_directory LOCAL_CORE_ENFORCER_DIR - fi - - if [[ -n "${LOCAL_DEPLOY_TOOL_DIR}" && "${COPY_LOCAL_DEPLOY_TOOL_DIR}" == "true" ]]; then - copy_local_dir_to_server_directory LOCAL_DEPLOY_TOOL_DIR - fi -} - -function copy_local_dir_to_server_directory { - local env_var=$1 - local src_dir=${!env_var} - local target_dir=$(basename $src_dir) - echo "Copying $env_var $src_dir into ./$target_dir" - if [[ -d $target_dir ]]; then - rm -rf $target_dir - fi - rsync -r $src_dir/* $target_dir --exclude node_modules --exclude build --exclude dist --exclude bin - tmp_dirs+=($target_dir) - export $env_var=$target_dir -} - -function build_docker_image { - if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then - build_cdn_enforcer_docker_image - else - build_enforcer_docker_image - fi -} - -function build_cdn_enforcer_docker_image { - container_name=${CONTAINER_NAME:-sample-${enforcer}-${cdn_container_type}} - container_tag=${CONTAINER_TAG:-latest} - - if [[ $cdn_container_type == "origin" ]]; then - build_cdn_origin - elif [[ $cdn_container_type == "test_endpoints" ]]; then - build_cdn_test_endpoints - elif [[ $cdn_container_type == "update_sample_site" ]]; then - echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" - build_cdn_update_sample_site - else - echo "No known docker file for CDN enforcer $enforcer: $cdn_container_type" - fi -} - -function build_cdn_origin { - echo "Building docker image $container_name:$container_tag" - docker build -t $container_name:$container_tag -f ./templates/origin/Dockerfile \ - --build-arg ENFORCER_NAME=$enforcer \ - --build-arg PORT=${PORT} \ - . -} - -function build_cdn_test_endpoints { - echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" - docker build -t $container_name:$container_tag -f ./servers/$enforcer/$cdn_container_type/Dockerfile \ - --build-arg ENFORCER_NAME=$enforcer \ - --build-arg PORT=${PORT} \ - --build-arg LOCAL_ENFORCER_DIR=${LOCAL_ENFORCER_DIR} \ - --build-arg LOCAL_CORE_ENFORCER_DIR=${LOCAL_CORE_ENFORCER_DIR} \ - --build-arg LOCAL_DEPLOY_TOOL_DIR=${LOCAL_DEPLOY_TOOL_DIR} \ - --build-arg PX_APP_ID=${PX_APP_ID} \ - --build-arg PX_AUTH_TOKEN=${PX_AUTH_TOKEN} \ - --build-arg PX_COOKIE_SECRET=${PX_COOKIE_SECRET} \ - . -} - -function build_cdn_update_sample_site { - echo "NOT YET READY" -} - -function build_enforcer_docker_image { - container_name=${CONTAINER_NAME:-sample-${enforcer}} - container_tag=${CONTAINER_TAG:-latest} - - echo "Building docker image $container_name:$container_tag using local enforcer ${LOCAL_ENFORCER_DIR}" - - docker build -t $container_name:$container_tag \ - --build-arg PX_APP_ID=${PX_APP_ID} \ - --build-arg PX_AUTH_TOKEN=${PX_AUTH_TOKEN} \ - --build-arg PX_COOKIE_SECRET=${PX_COOKIE_SECRET} \ - --build-arg ENABLE_TEST_ENDPOINTS=${ENABLE_TEST_ENDPOINTS:-true} \ - --build-arg LOCAL_ENFORCER_DIR=${LOCAL_ENFORCER_DIR} \ - --build-arg LOCAL_CORE_ENFORCER_DIR=${LOCAL_CORE_ENFORCER_DIR} \ - --build-arg DOCKER_ENFORCER_DIR=${DOCKER_ENFORCER_DIR} \ - -f servers/$enforcer/Dockerfile . -} - -function remove_tmp_enforcer_directories { - for tmp_dir in "${tmp_dirs[@]}" - do - echo "Removing ./$tmp_dir" - rm -rf $tmp_dir - done -} - -main $@ \ No newline at end of file diff --git a/demo-site/scripts/common.sh b/demo-site/scripts/common.sh deleted file mode 100644 index afb9578..0000000 --- a/demo-site/scripts/common.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -set -e - -shared_config_json_file=./shared_config.json -cdn_enforcers="fastly:cloudflare:lambda:akamai:fastly_js" -tmp_dirs=() - -function is_cdn_enforcer { - echo "$(contains $cdn_enforcers $1)" -} - -function contains { - [[ ":$1:" =~ ":$2:" ]] && echo "true" || echo "false" -} - -function validate_and_set_args { - if [[ -z $1 ]]; then - echo "For which server?" - read enforcer - else - enforcer=$1 - fi - - if [[ ! -d "./servers/$enforcer" ]]; then - echo "No directory found for enforcer $enforcer, exiting..." - exit 1 - fi - - config_json_file=./servers/$enforcer/config.json - if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then - if [[ -z $2 || $2 != "origin" && $2 != "test_endpoints" && $2 != "update_sample_site" ]]; then - echo "origin, test_endpoints, or update_sample_site?" - read cdn_container_type - else - cdn_container_type=$2 - fi - - if [[ ! -d "./servers/$enforcer/$cdn_container_type" ]]; then - echo "No directory $cdn_container_type found for enforcer $enforcer, exiting..." - exit 1 - fi - fi -} - -function get_value_from_config { - desired_value=$1 - json_file=$2 - if [[ ! -f $json_file ]]; then - echo "" - return - fi - value=$(cat $json_file | grep $desired_value | cut -d ":" -f2 | tr -d '\ \," "') - echo $value -} - -function use_env_or_config_value { - env_var=$1 - config_key=$2 - required=$3 - if [[ -z ${!env_var} ]]; then - export $env_var="$(get_value_from_config $config_key $config_json_file)" - fi - - if [[ -z ${!env_var} ]]; then - export $env_var="$(get_value_from_config $config_key $shared_config_json_file)" - fi - - if [[ "$required" == "true" && -z ${!env_var} ]]; then - echo "Unable to initialize mandatory env variable ${env_var}!" - exit 1 - fi -} \ No newline at end of file diff --git a/demo-site/scripts/run_docker.sh b/demo-site/scripts/run_docker.sh deleted file mode 100755 index 5f9ac80..0000000 --- a/demo-site/scripts/run_docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -e - -function main { - source ./scripts/common.sh - - validate_and_set_args $@ - - set_variables $@ - - run_docker_container -} - -function set_variables { - if [[ "$(is_cdn_enforcer $enforcer)" == "true" ]]; then - container_name=${CONTAINER_NAME:-sample-${enforcer}-${cdn_container_type}} - else - container_name=${CONTAINER_NAME:-sample-${enforcer}} - fi - - container_tag=${CONTAINER_TAG:-latest} - port=${PORT:-3000} - - if [[ "$2" == "mount" ]]; then - should_mount=true - fi -} - -function run_docker_container { - echo "Running $container_name:$container_tag" - if [[ "$should_mount" == "true" ]]; then - mount_and_run_docker - else - run_docker - fi -} - -function mount_and_run_docker { - use_env_or_config_value LOCAL_ENFORCER_DIR local_enforcer_dir true - use_env_or_config_value DOCKER_ENFORCER_DIR docker_enforcer_dir true - use_env_or_config_value LOCAL_CORE_ENFORCER_DIR local_core_enforcer_dir false - use_env_or_config_value DOCKER_CORE_ENFORCER_DIR docker_core_enforcer_dir false - - if [[ -n "$LOCAL_CORE_ENFORCER_DIR" || -n "$DOCKER_CORE_ENFORCER_DIR" ]]; then - echo "mounting local enforcer dir ${LOCAL_ENFORCER_DIR} and local core enforcer dir ${LOCAL_CORE_ENFORCER_DIR}" - docker run -it -p $port:$port \ - -v ${LOCAL_ENFORCER_DIR}:${DOCKER_ENFORCER_DIR} \ - -v ${LOCAL_CORE_ENFORCER_DIR}:${DOCKER_CORE_ENFORCER_DIR} \ - $container_name:$container_tag - else - echo "mounting local enforcer dir ${LOCAL_ENFORCER_DIR}" - docker run -it -p $port:$port \ - -v ${LOCAL_ENFORCER_DIR}:${DOCKER_ENFORCER_DIR} \ - $container_name:$container_tag - fi -} - -function run_docker { - docker run -it -p $port:$port $container_name:$container_tag -} - -main $@ \ No newline at end of file diff --git a/demo-site/templates/origin/Dockerfile b/demo-site/templates/origin/Dockerfile deleted file mode 100644 index 9580996..0000000 --- a/demo-site/templates/origin/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM node:16-alpine as builder -ARG ENFORCER_NAME - -WORKDIR /tmp/sample-sites - -COPY shared_config.json . -COPY scripts scripts -COPY utils utils -COPY templates/static_files templates/static_files -COPY servers/${ENFORCER_NAME} servers/${ENFORCER_NAME} - -RUN node scripts/create_static_files.js && node scripts/create_px_configs.js - -FROM node:16-alpine as origin -ARG ENFORCER_NAME - -WORKDIR /app -COPY utils utils - -WORKDIR /app/templates/origin -COPY templates/origin/package.json . -RUN npm install -COPY templates/origin/*.js . - -WORKDIR /app/servers/${ENFORCER_NAME} -COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/px_config.json . -COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/config.inc.json . - -WORKDIR /app/servers/${ENFORCER_NAME}/origin -COPY --from=builder /tmp/sample-sites/servers/${ENFORCER_NAME}/origin/public/ public/ -COPY servers/${ENFORCER_NAME}/origin/package.json . -RUN npm install -COPY servers/${ENFORCER_NAME}/origin/*.js . - -ARG PORT=3000 -ENV PORT ${PORT} - -EXPOSE ${PORT} -CMD ["npm", "start"] \ No newline at end of file diff --git a/demo-site/templates/origin/origin_app.js b/demo-site/templates/origin/origin_app.js deleted file mode 100644 index db5553c..0000000 --- a/demo-site/templates/origin/origin_app.js +++ /dev/null @@ -1,73 +0,0 @@ -const path = require('path'); -const { env } = require('process'); -const express = require('express'); -const cookieParser = require('cookie-parser'); -const { - INDEX_ROUTE, - LOGIN_ROUTE, - LOGOUT_ROUTE, - EXPECTED_USERNAME, - EXPECTED_PASSWORD, - FIRST_PARTY_STATIC_FILE_SUFFIX, - THIRD_PARTY_STATIC_FILE_SUFFIX -} = require("../../utils/constants"); - -class OriginApp { - constructor(enforcerType) { - this.enforcerType = enforcerType; - this.app = this.createApp(); - } - - createApp() { - const app = express(); - app.use(cookieParser()); - app.use(express.json()); - app.use(express.urlencoded({ extended: true })); - app.use((req, res, next) => { - console.log(`${req.method} ${req.path}`); - res.set(req.headers); - next(); - }) - return app; - } - - SetRoutes(staticRoutesDir, isFirstParty) { - this.app.use(express.static(staticRoutesDir, {index: false})); - - const htmlSuffix = isFirstParty ? - FIRST_PARTY_STATIC_FILE_SUFFIX : THIRD_PARTY_STATIC_FILE_SUFFIX; - - this.app.get(INDEX_ROUTE, (req, res) => { - const htmlPageName = `index${htmlSuffix}.html`; - res.sendFile(path.join(staticRoutesDir, htmlPageName)); - }); - - this.app.post(LOGIN_ROUTE, (req, res) => { - if (req.body.username === EXPECTED_USERNAME && - req.body.password === EXPECTED_PASSWORD) { - res.sendFile(path.join(staticRoutesDir, `profile${htmlSuffix}.html`)); - } else { - res.redirect(INDEX_ROUTE); - } - }); - - this.app.get(LOGOUT_ROUTE, (req, res) => { - res.redirect(INDEX_ROUTE); - }); - } - - Start() { - const port = env.PORT || 3000; - const server = this.app.listen(port, '0.0.0.0', () => { - console.log(`${this.enforcerType} origin listening on port ${port}!`) - }); - - process.on('SIGINT', () => { - console.log('\nClosing http server...'); - server.close(); - process.exit(0); - }); - } -} - -module.exports = OriginApp; diff --git a/demo-site/templates/origin/package.json b/demo-site/templates/origin/package.json deleted file mode 100644 index 4c19da8..0000000 --- a/demo-site/templates/origin/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "origin-utils", - "version": "1.0.0", - "description": "", - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "cookie-parser": "^1.4.5", - "express": "^4.17.1" - } -} diff --git a/demo-site/templates/test_endpoints/package.json b/demo-site/templates/test_endpoints/package.json deleted file mode 100644 index 9d4988e..0000000 --- a/demo-site/templates/test_endpoints/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "test-endpoints-utils", - "version": "1.0.0", - "description": "", - "author": "", - "license": "ISC", - "dependencies": { - "cookie-parser": "^1.4.3", - "express": "^4.14.0" - }, - "devDependencies": {} -} diff --git a/demo-site/templates/test_endpoints/test_endpoints_app.js b/demo-site/templates/test_endpoints/test_endpoints_app.js deleted file mode 100644 index a0bfee3..0000000 --- a/demo-site/templates/test_endpoints/test_endpoints_app.js +++ /dev/null @@ -1,94 +0,0 @@ -const path = require('path'); -const express = require('express'); -const cookieParser = require('cookie-parser'); -const { env } = require('process'); -const { loadJson } = require("../../utils/utils"); -const { - TEST_APP_CREDENTIALS_ENDPOINT, - SUPPORTED_FEATURES_ENDPOINT, - CONFIG_ENDPOINT, - PX_METADATA_FILE_NAME -} = require('../../utils/constants'); - -class TestEndpointsApp { - constructor(enforcerType) { - this.enforcerType = enforcerType; - this.app = this.createApp(); - } - - createApp() { - const app = express(); - app.use(cookieParser()); - app.use(express.json()); - app.use(express.urlencoded({ - extended: true - })); - return app; - } - - SetTestAppCredentialsEndpoint({ px_app_id, px_cookie_secret }) { - this.app.get(TEST_APP_CREDENTIALS_ENDPOINT, (req, res) => { - console.log(`GET ${TEST_APP_CREDENTIALS_ENDPOINT}`); - const credentials = { - px_app_id, - px_cookie_secret - }; - res.json(credentials); - }); - } - - SetSupportedFeaturesEndpoint(enforcerDir) { - this.app.get(SUPPORTED_FEATURES_ENDPOINT, (req, res) => { - console.log(`GET ${SUPPORTED_FEATURES_ENDPOINT}`); - try { - const pxMetadata = loadJson(path.join(enforcerDir, PX_METADATA_FILE_NAME)); - res.json(pxMetadata); - } catch (err) { - console.error(err); - res.sendStatus(500); - } - }); - } - - /** - * @param changeEnforcerConfigCallback: function (enforcerConfig, serverConfig) => boolean - * the return value indicates whether the enforcer config was successfuly updated - * - * @param serverConfig - object with structure identical to config.inc.json - * fields are filled in with either the JSON values or env variables - */ - SetConfigEndpoint(changeEnforcerConfigCallback, serverConfig) { - this.app.post(CONFIG_ENDPOINT, async (req, res) => { - console.log(`POST ${CONFIG_ENDPOINT}`); - try { - if (!changeEnforcerConfigCallback) { - console.log("Skipping changing enforcer config..."); - return res.sendStatus(200); - } - if (!(await changeEnforcerConfigCallback(req.body, serverConfig))) { - console.error("Unable to change enforcer config!"); - return res.sendStatus(500); - } - res.sendStatus(200); - } catch (err) { - console.error(err); - res.sendStatus(500); - } - }); - } - - Start() { - const port = env.PORT || 3000; - const server = this.app.listen(port, '0.0.0.0', () => { - console.log(`${this.enforcerType} test endpoints listening on port ${port}!`); - }); - - process.on('SIGINT', () => { - console.log('\nClosing http server...'); - server.close(); - process.exit(0); - }); - } -} - -module.exports = TestEndpointsApp; \ No newline at end of file diff --git a/demo-site/utils/cdn_deploy_tool_utils.js b/demo-site/utils/cdn_deploy_tool_utils.js deleted file mode 100644 index d50ca5e..0000000 --- a/demo-site/utils/cdn_deploy_tool_utils.js +++ /dev/null @@ -1,82 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); -const { loadJson, saveJson } = require("./utils"); -const { - CDN_DEPLOY_TOOL_RUN_CLI_COMMAND, - CDN_DEPLOY_TOOL_CONFIG_FILE_NAME, - CDN_DEPLOY_TOOL_BUILD_COMMAND -} = require("./constants"); - -const setCDNDeployToolPipeline = (deployToolPath, cdnType, deployType) => { - const deployToolConfigPath = path.join(deployToolPath, "configs", CDN_DEPLOY_TOOL_CONFIG_FILE_NAME); - const deployToolConfig = loadJson(deployToolConfigPath); - if (!deployToolConfig) { - console.error(`Cannot find ${deployToolConfigPath}!`); - return false; - } - - deployToolConfig.cdnType = cdnType; - deployToolConfig.deployType = deployType; - - saveJson(deployToolConfigPath, deployToolConfig); - return true; -}; - -const getCDNDeployToolPipelineConfig = (deployToolPath, pipelineName) => { - const pipelineConfigIncPath = path.join(deployToolPath, `configs/pipelines/${pipelineName}/pipelineConfig.inc.json`); - const pipelineConfigIncJson = loadJson(pipelineConfigIncPath); - if (pipelineConfigIncJson) { - return pipelineConfigIncJson; - } - console.error(`Cannot find ${pipelineConfigIncPath}!`); - return null; -}; - -const setCDNDeployToolPipelineConfig = (deployToolPath, pipelineName, pipelineConfig) => { - const pipelineConfigPath = path.join(deployToolPath, `configs/pipelines/${pipelineName}/pipelineConfig.json`); - saveJson(pipelineConfigPath, pipelineConfig); - return true; -} - -const runCDNDeployTool = (deployToolPath) => { - if (!isCDNDeployToolReady(deployToolPath)) { - return false; - } - console.log("\nRunning deploy pipeline"); - if (!runCDNDeployToolCommand(deployToolPath, CDN_DEPLOY_TOOL_RUN_CLI_COMMAND)) { - return false; - } - console.log("Deploy completed successfully!"); - return true; -} - -const isCDNDeployToolReady = (deployToolPath) => { - if (!fs.existsSync(path.join(deployToolPath, "node_modules"))) { - console.error("CDN Deploy tool dependencies are not installed"); - return false; - } - if (!fs.existsSync(path.join(deployToolPath, "build"))) { - console.info("CDN Deploy tool has not been compiled"); - if (!runCDNDeployToolCommand(deployToolPath, CDN_DEPLOY_TOOL_BUILD_COMMAND)) { - return false; - } - } - return true; -} - -const runCDNDeployToolCommand = (deployToolPath, command) => { - const output = execSync(command, { cwd: deployToolPath }); - console.log(output + '\n'); - let outputString = output.toString(); - console.log(outputString); - outputString = outputString.toLowerCase(); - return outputString.indexOf("error") === -1 && outputString.indexOf("fail") === -1; -} - -module.exports = { - setCDNDeployToolPipeline, - getCDNDeployToolPipelineConfig, - setCDNDeployToolPipelineConfig, - runCDNDeployTool -} \ No newline at end of file From 418aa8c51dba7f94d639fb9e2310472863396162 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Mon, 3 Jul 2023 19:24:28 +0300 Subject: [PATCH 04/13] fix comments after pr --- .github/workflows/E2E_CI.yaml | 37 +++++++++++++++++++++++------------ ci_files/build_cluster.sh | 9 --------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/E2E_CI.yaml b/.github/workflows/E2E_CI.yaml index 6732331..fe0f3fa 100644 --- a/.github/workflows/E2E_CI.yaml +++ b/.github/workflows/E2E_CI.yaml @@ -32,7 +32,7 @@ jobs: - name: build local cluster uses: actions/checkout@v2 - - run: ./ci_files/build_cluster.sh + run: ./ci_files/build_cluster.sh - name: Set up Docker uses: docker/setup-buildx-action@v1 @@ -50,23 +50,37 @@ jobs: sudo apt-get install apt-transport-https --yes echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list sudo apt-get update - sudo apt-get install helm + sudo apt-get install helm=3.12.1-1 - - name: Checkout enforcer repo + - name: Clone helm charts repo - mock-collector uses: actions/checkout@v2 + with: + repository: PerimeterX/connect-helm-charts + token: ${{ secrets.CONNECT_PULL_TOKEN }} + ref: mock-collector-0.1.0 + path: ./deploy_charts/mock-collector - - name: Clone helm charts repo + - name: Clone helm charts repo - enforcer-tests uses: actions/checkout@v2 with: repository: PerimeterX/connect-helm-charts token: ${{ secrets.CONNECT_PULL_TOKEN }} - ref: main - path: ./deploy_charts + ref: enforcer-spec-tests-0.6.0 + path: ./deploy_charts/enforcer-spec-tests + + + - name: Clone helm charts repo - sample-site + uses: actions/checkout@v2 + with: + repository: PerimeterX/connect-helm-charts + token: ${{ secrets.CONNECT_PULL_TOKEN }} + ref: sample-site-0.1.0 + path: ./deploy_charts/sample-site - name: deploy sample site run: | - helm install sample-site ./deploy_charts/charts/sample-site --set image.name=localhost:5001/node-sample-site --set image.tag=1.0.0 --set imagePullPolicy=Always --set collectorURL=http://mock-collector-mock-collector:3001 --wait + helm install sample-site ./deploy_charts/sample-site/charts/sample-site --set image.name=localhost:5001/node-sample-site --set image.tag=1.0.0 --set imagePullPolicy=Always --set collectorURL=http://mock-collector-mock-collector:3001 --wait - name: Set up Google Cloud SDK id: 'auth' @@ -87,7 +101,7 @@ jobs: - name: deploy mock collector run: | - helm install mock-collector ./deploy_charts/charts/mock-collector --set image.repository=localhost:5001/mock-collector --set image.tag=1.0.2 --set imagePullPolicy=Always --wait + helm install mock-collector ./deploy_charts/mock-collector/charts/mock-collector --set image.repository=localhost:5001/mock-collector --set image.tag=1.0.2 --set imagePullPolicy=Always --wait - run: kubectl get pods @@ -100,7 +114,7 @@ jobs: - name: run enforcer tests run: | - helm install enforcer-spec-tests ./deploy_charts/charts/enforcer-spec-tests --set image.repository=localhost:5001/enforcer-spec-tests --set image.tag=1.1.0 --set imagePullPolicy=Always \ + helm install enforcer-spec-tests ./deploy_charts/enforcer-spec-tests/charts/enforcer-spec-tests --set image.repository=localhost:5001/enforcer-spec-tests --set image.tag=1.1.0 --set imagePullPolicy=Always \ --set internalMockCollectorURL=http://mock-collector-mock-collector:3001 \ --set appID=PXnEpdw6lS \ --set siteURL=http://sample-site-sample-site:3000 \ @@ -114,8 +128,5 @@ jobs: JOB_NAME: enforcer-spec-tests - name: get tests results - if: ${{ failure() }} - run: kubectl logs job/enforcer-spec-tests - - - name: get tests results + if: ${{ always() }} run: kubectl logs job/enforcer-spec-tests \ No newline at end of file diff --git a/ci_files/build_cluster.sh b/ci_files/build_cluster.sh index 5b87aa6..b48715f 100755 --- a/ci_files/build_cluster.sh +++ b/ci_files/build_cluster.sh @@ -1,15 +1,6 @@ #!/bin/sh set -o errexit -# 1. Download kind binary -# For AMD64 / x86_64 -#[ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.19.0/kind-linux-amd64 -# For ARM64 -#[ $(uname -m) = aarch64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.19.0/kind-linux-arm64 -#chmod +x ./kind -#sudo mv ./kind /usr/local/bin/kind - - # 2. Create registry container unless it already exists reg_name='kind-registry' reg_port='5001' From 386ae63cae65594a75d5f1ab87999aa4670965d1 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Mon, 3 Jul 2023 19:25:34 +0300 Subject: [PATCH 05/13] fix comments after pr --- .github/workflows/E2E_CI.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/E2E_CI.yaml b/.github/workflows/E2E_CI.yaml index fe0f3fa..e583831 100644 --- a/.github/workflows/E2E_CI.yaml +++ b/.github/workflows/E2E_CI.yaml @@ -30,8 +30,10 @@ jobs: steps: - - name: build local cluster + - name: Checkout Repo uses: actions/checkout@v2 + + - name: Build local cluster run: ./ci_files/build_cluster.sh - name: Set up Docker From 3769ac58b019fb62132b989b664cf6e5242c5a3f Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Wed, 19 Jul 2023 15:02:05 +0300 Subject: [PATCH 06/13] fix comments after pr --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c2d3d65..0cf0030 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ COPY ./demo-site/servers/nodejs/package.json servers/nodejs/package.json RUN cd servers/nodejs && npm install COPY ./demo-site/servers/nodejs servers/nodejs -RUN node scripts/create_static_files.js && node scripts/create_px_configs.js +RUN node scripts/create_static_files.js WORKDIR /workspace/servers/nodejs From 1cc6783c6e8b128b3deffadda34cc3611aff3253 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Wed, 19 Jul 2023 15:14:22 +0300 Subject: [PATCH 07/13] fix comments after pr --- Dockerfile | 2 -- demo-site/scripts/create_px_configs.js | 22 ---------------------- 2 files changed, 24 deletions(-) delete mode 100644 demo-site/scripts/create_px_configs.js diff --git a/Dockerfile b/Dockerfile index 0cf0030..fcc2855 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,6 @@ COPY ./demo-site/servers/nodejs/package.json servers/nodejs/package.json RUN cd servers/nodejs && npm install COPY ./demo-site/servers/nodejs servers/nodejs -RUN node scripts/create_static_files.js - WORKDIR /workspace/servers/nodejs COPY ./ perimeterx-node-express diff --git a/demo-site/scripts/create_px_configs.js b/demo-site/scripts/create_px_configs.js deleted file mode 100644 index f7f8469..0000000 --- a/demo-site/scripts/create_px_configs.js +++ /dev/null @@ -1,22 +0,0 @@ -// region imports -const path = require('path'); -const sharedConfig = require('../shared_config.json'); -const { forEachServer, sortObjectAlphabeticallyByKey, capitalize, saveJson } = require('../utils/utils'); -const { PX_ENFORCER_CONFIG_FILE_NAME } = require('../utils/constants'); -// endregion - -const main = () => { - forEachServer((serverName, serverPath, serverConfig) => { - const pxConfig = mergeConfigs(serverConfig, sharedConfig); - saveJson(path.join(serverPath, `${PX_ENFORCER_CONFIG_FILE_NAME}`), pxConfig); - console.log(`Successfully created ${PX_ENFORCER_CONFIG_FILE_NAME} for ${capitalize(serverName)}`); - }); -}; - -const mergeConfigs = (serverConfig, sharedConfig) => { - const { enforcer_credentials, enforcer_config_override } = serverConfig; - const { enforcer_config } = sharedConfig; - return sortObjectAlphabeticallyByKey(Object.assign({}, enforcer_config, enforcer_credentials, enforcer_config_override)); -}; - -main(); From 99f494fadbf60f68a74fbb7fcb25eb374ace59b2 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Wed, 19 Jul 2023 15:28:06 +0300 Subject: [PATCH 08/13] fix comments after pr --- Dockerfile | 1 - demo-site/scripts/clean_static_files.js | 58 ------------- demo-site/scripts/create_static_files.js | 105 ----------------------- px_metadata.json | 2 - 4 files changed, 166 deletions(-) delete mode 100644 demo-site/scripts/clean_static_files.js delete mode 100644 demo-site/scripts/create_static_files.js diff --git a/Dockerfile b/Dockerfile index fcc2855..9b9166e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM node:16-slim WORKDIR /workspace COPY ./demo-site/shared_config.json . -COPY ./demo-site/scripts scripts COPY ./demo-site/templates templates COPY ./demo-site/utils utils COPY ./demo-site/servers/nodejs/package.json servers/nodejs/package.json diff --git a/demo-site/scripts/clean_static_files.js b/demo-site/scripts/clean_static_files.js deleted file mode 100644 index 03becd4..0000000 --- a/demo-site/scripts/clean_static_files.js +++ /dev/null @@ -1,58 +0,0 @@ -//region imports -const fs = require('fs'); -const path = require('path'); -const process = require('process'); -const { forEachServer, getUserInput, capitalize } = require("../utils/utils"); -const { CONFIG_FILE_NAME } = require('../utils/constants'); -//endregion - -const FILE_TYPES_TO_DELETE = [".html", ".css"]; - -const main = async () => { - if (!(await confirmDeletion())) { - console.log("Public directories will not be deleted."); - return; - } - - forEachServer((serverName, serverPath, serverConfig) => { - cleanStaticFiles(serverPath, serverConfig); - console.log(`Cleaned static files for ${capitalize(serverName)}`); - }); -}; - -const confirmDeletion = async () => { - const FLAG_INDEX = 2; - const CONFIRM_DELETION_FLAG = "-y"; - const YES_ANSWERS = ["yes", "y"]; - - if (process.argv[FLAG_INDEX] === CONFIRM_DELETION_FLAG) { - return true; - } - - const response = await getUserInput("Are you sure you want to delete static files in all servers?"); - return YES_ANSWERS.includes(response.toLowerCase()); -}; - -const cleanStaticFiles = (serverPath, config) => { - if (!config.site_config || !config.site_config.public_output_dir) { - console.error(`No property "public_output_dir" in ${serverPath}/${CONFIG_FILE_NAME}.json: ${config}`); - process.exit(1); - } - - if (config.site_config.public_output_dir === ".") { - fs.readdirSync(serverPath).forEach((filename) => { - for (const filetype of FILE_TYPES_TO_DELETE) { - if (filename.endsWith(filetype)) { - fs.rmSync(path.join(serverPath, filename)); - } - } - }); - } else { - const publicFilesPath = path.join(serverPath, config.site_config.public_output_dir); - if (fs.existsSync(publicFilesPath)) { - fs.rmdirSync(publicFilesPath, { recursive: true, force: true }); - } - } -}; - -main(); \ No newline at end of file diff --git a/demo-site/scripts/create_static_files.js b/demo-site/scripts/create_static_files.js deleted file mode 100644 index 000ee92..0000000 --- a/demo-site/scripts/create_static_files.js +++ /dev/null @@ -1,105 +0,0 @@ -//region imports -const fs = require('fs'); -const path = require('path'); -const sharedConfig = require('../shared_config.json'); -const { forEachServer, normalizeEnforcerName } = require("../utils/utils"); -const { CONFIG_FILE_NAME } = require('../utils/constants'); -//endregion - -const TEMPLATE_INDICATOR = sharedConfig.site_config.template_indicator; -const REPLACE_VARIABLE_REGEX = /\${[A-Za-z1-9_]*}/gi; - -const main = () => { - const publicTemplateDir = path.join(__dirname, "..", sharedConfig.site_config.public_template_dir); - forEachServer((serverName, serverPath, serverConfig) => { - const publicDir = createPublicDirForServer(serverPath, serverConfig); - if (!publicDir) { - console.error(`Couldn't create public dir for ${serverName} server`); - process.exit(1); - } - - if (!copyStaticFiles(publicTemplateDir, serverConfig, serverName, publicDir)) { - console.error(`Could not copy static files for ${normalizeEnforcerName(serverName)}. Skipping...`); - } else { - console.log(`Successfully created static files for ${normalizeEnforcerName(serverName)}`); - } - }); -}; - -const copyStaticFiles = (publicTemplateDir, serverConfig, serverName, publicDir) => { - const replaceVariableMaps = createReplacementVariableMaps(serverConfig, serverName); - if (!replaceVariableMaps) { - return false; - } - - const fileNames = fs.readdirSync(publicTemplateDir); - for (const fileName of fileNames) { - copyFileToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); - } - return true; -}; - -const copyFileToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { - if (fileName.includes(TEMPLATE_INDICATOR)) { - copyTemplateToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); - } else { - copyStaticFileToServerDirectory(fileName, publicTemplateDir, publicDir); - } -}; - -const copyTemplateToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { - const template = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); - for (const { templateIndicatorReplacement, replacementMap } of replaceVariableMaps) { - const fileContents = fillInTemplate(template, replacementMap); - const newFileName = fileName.replace(TEMPLATE_INDICATOR, templateIndicatorReplacement); - fs.writeFileSync(path.join(publicDir, newFileName), fileContents); - } -}; - -const copyStaticFileToServerDirectory = (fileName, publicTemplateDir, publicDir) => { - const fileContents = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); - fs.writeFileSync(path.join(publicDir, fileName), fileContents); -}; - -const fillInTemplate = (template, replaceVariableMap) => { - return template.replace(REPLACE_VARIABLE_REGEX, (matched) => replaceVariableMap[matched]); -} - -const createReplacementVariableMaps = (config, enforcerName) => { - const appId = config.enforcer_credentials.px_app_id; - if (appId == null || appId.length === 0) { - console.error(`No px_app_id found in ${enforcerName}/${CONFIG_FILE_NAME}!`); - return null; - } - const appIdSubstr = appId.substr(2); - return [ - createReplacementInfo("", appId, `//client.px-cloud.net/${appId}/main.min.js`, enforcerName), - createReplacementInfo(".firstparty", appId, `/${appIdSubstr}/init.js`, enforcerName) - ]; -}; - -const createReplacementInfo = (templateIndicatorReplacement, appId, sensorSrcUrl, enforcerName) => { - return { - templateIndicatorReplacement, - replacementMap: { - "${app_id}": appId, - "${sensor_src_url}": sensorSrcUrl, - "${enforcer_name}": normalizeEnforcerName(enforcerName) - } - }; -}; - -const createPublicDirForServer = (serverPath, config) => { - if (!config.site_config || !config.site_config.public_output_dir) { - console.error(`No property "public_output_dir" in ${serverPath}/${CONFIG_FILE_NAME}: ${config}`); - return null; - } - - const publicFilesPath = path.join(serverPath, config.site_config.public_output_dir); - if (!fs.existsSync(publicFilesPath)) { - fs.mkdirSync(publicFilesPath); - } - return publicFilesPath; -}; - -main(); diff --git a/px_metadata.json b/px_metadata.json index cf5bf4d..392c83f 100644 --- a/px_metadata.json +++ b/px_metadata.json @@ -6,7 +6,6 @@ "batched_activities", "block_activity", "block_page_captcha", - "block_page_hard_block", "block_page_rate_limit", "block_page_js_challenge", "bypass_monitor_header", @@ -56,7 +55,6 @@ "test_vid_extraction_on_first_party_xhr", "test_block_activity_cookie_origin", "test_page_requested_activity_cookie_origin", - "test_block_page_hard_block_response", "test_risk_api_validate_cookie_origin", "test_risk_cookie_valid_cookie_with_user_agent_bigger_than_max_length" ] From 900e4f747d5ee78f8b9c509cfaf31d77a8440bb3 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Wed, 19 Jul 2023 15:46:42 +0300 Subject: [PATCH 09/13] fix comments after pr --- Dockerfile | 3 + demo-site/scripts/create_static_files.js | 105 +++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 demo-site/scripts/create_static_files.js diff --git a/Dockerfile b/Dockerfile index 9b9166e..9c3586d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,13 @@ WORKDIR /workspace COPY ./demo-site/shared_config.json . COPY ./demo-site/templates templates COPY ./demo-site/utils utils +COPY ./demo-site/scripts scripts COPY ./demo-site/servers/nodejs/package.json servers/nodejs/package.json RUN cd servers/nodejs && npm install COPY ./demo-site/servers/nodejs servers/nodejs +RUN node scripts/create_static_files.js + WORKDIR /workspace/servers/nodejs COPY ./ perimeterx-node-express diff --git a/demo-site/scripts/create_static_files.js b/demo-site/scripts/create_static_files.js new file mode 100644 index 0000000..000ee92 --- /dev/null +++ b/demo-site/scripts/create_static_files.js @@ -0,0 +1,105 @@ +//region imports +const fs = require('fs'); +const path = require('path'); +const sharedConfig = require('../shared_config.json'); +const { forEachServer, normalizeEnforcerName } = require("../utils/utils"); +const { CONFIG_FILE_NAME } = require('../utils/constants'); +//endregion + +const TEMPLATE_INDICATOR = sharedConfig.site_config.template_indicator; +const REPLACE_VARIABLE_REGEX = /\${[A-Za-z1-9_]*}/gi; + +const main = () => { + const publicTemplateDir = path.join(__dirname, "..", sharedConfig.site_config.public_template_dir); + forEachServer((serverName, serverPath, serverConfig) => { + const publicDir = createPublicDirForServer(serverPath, serverConfig); + if (!publicDir) { + console.error(`Couldn't create public dir for ${serverName} server`); + process.exit(1); + } + + if (!copyStaticFiles(publicTemplateDir, serverConfig, serverName, publicDir)) { + console.error(`Could not copy static files for ${normalizeEnforcerName(serverName)}. Skipping...`); + } else { + console.log(`Successfully created static files for ${normalizeEnforcerName(serverName)}`); + } + }); +}; + +const copyStaticFiles = (publicTemplateDir, serverConfig, serverName, publicDir) => { + const replaceVariableMaps = createReplacementVariableMaps(serverConfig, serverName); + if (!replaceVariableMaps) { + return false; + } + + const fileNames = fs.readdirSync(publicTemplateDir); + for (const fileName of fileNames) { + copyFileToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); + } + return true; +}; + +const copyFileToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { + if (fileName.includes(TEMPLATE_INDICATOR)) { + copyTemplateToServerDirectory(fileName, publicTemplateDir, replaceVariableMaps, publicDir); + } else { + copyStaticFileToServerDirectory(fileName, publicTemplateDir, publicDir); + } +}; + +const copyTemplateToServerDirectory = (fileName, publicTemplateDir, replaceVariableMaps, publicDir) => { + const template = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); + for (const { templateIndicatorReplacement, replacementMap } of replaceVariableMaps) { + const fileContents = fillInTemplate(template, replacementMap); + const newFileName = fileName.replace(TEMPLATE_INDICATOR, templateIndicatorReplacement); + fs.writeFileSync(path.join(publicDir, newFileName), fileContents); + } +}; + +const copyStaticFileToServerDirectory = (fileName, publicTemplateDir, publicDir) => { + const fileContents = fs.readFileSync(path.join(publicTemplateDir, fileName)).toString(); + fs.writeFileSync(path.join(publicDir, fileName), fileContents); +}; + +const fillInTemplate = (template, replaceVariableMap) => { + return template.replace(REPLACE_VARIABLE_REGEX, (matched) => replaceVariableMap[matched]); +} + +const createReplacementVariableMaps = (config, enforcerName) => { + const appId = config.enforcer_credentials.px_app_id; + if (appId == null || appId.length === 0) { + console.error(`No px_app_id found in ${enforcerName}/${CONFIG_FILE_NAME}!`); + return null; + } + const appIdSubstr = appId.substr(2); + return [ + createReplacementInfo("", appId, `//client.px-cloud.net/${appId}/main.min.js`, enforcerName), + createReplacementInfo(".firstparty", appId, `/${appIdSubstr}/init.js`, enforcerName) + ]; +}; + +const createReplacementInfo = (templateIndicatorReplacement, appId, sensorSrcUrl, enforcerName) => { + return { + templateIndicatorReplacement, + replacementMap: { + "${app_id}": appId, + "${sensor_src_url}": sensorSrcUrl, + "${enforcer_name}": normalizeEnforcerName(enforcerName) + } + }; +}; + +const createPublicDirForServer = (serverPath, config) => { + if (!config.site_config || !config.site_config.public_output_dir) { + console.error(`No property "public_output_dir" in ${serverPath}/${CONFIG_FILE_NAME}: ${config}`); + return null; + } + + const publicFilesPath = path.join(serverPath, config.site_config.public_output_dir); + if (!fs.existsSync(publicFilesPath)) { + fs.mkdirSync(publicFilesPath); + } + return publicFilesPath; +}; + +main(); From b30648b14d48a52f943ffdb8dedf5cea4c1d747f Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Wed, 19 Jul 2023 16:04:27 +0300 Subject: [PATCH 10/13] fix comments after pr --- demo-site/servers/nodejs/config.inc.json | 16 ---------------- demo-site/servers/nodejs/config.json | 23 ----------------------- demo-site/utils/constants.js | 2 +- 3 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 demo-site/servers/nodejs/config.inc.json delete mode 100644 demo-site/servers/nodejs/config.json diff --git a/demo-site/servers/nodejs/config.inc.json b/demo-site/servers/nodejs/config.inc.json deleted file mode 100644 index 8f10fe7..0000000 --- a/demo-site/servers/nodejs/config.inc.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "site_config": { - "public_output_dir": "public", - "local_enforcer_dir": "", - "local_core_enforcer_dir": "", - "port": 3000 - }, - "enforcer_credentials": { - "px_app_id": "PXZ7wl3dvS", - "px_auth_token": "", - "px_cookie_secret": "" - }, - "enforcer_config_override": { - - } -} \ No newline at end of file diff --git a/demo-site/servers/nodejs/config.json b/demo-site/servers/nodejs/config.json deleted file mode 100644 index b093a9d..0000000 --- a/demo-site/servers/nodejs/config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "site_config": { - "public_output_dir": "public", - "local_enforcer_dir": "../../", - "local_core_enforcer_dir": "", - "port": 3000 - }, - "enforcer_credentials": { - "px_app_id": "PXnEpdw6lS", - "px_auth_token": "", - "px_cookie_secret": "" - }, - "enforcer_config_override": { - "px_blocking_score": 70, - "px_module_mode": "active_blocking", - "px_s2s_timeout": 1000, - "px_user_agent_max_length": 8528, - "px_risk_cookie_max_length": 2048, - "px_risk_cookie_max_iterations": 5000, - "px_risk_cookie_min_iterations": 1, - "px_custom_cookie_header": "x-px-cookies" - } -} \ No newline at end of file diff --git a/demo-site/utils/constants.js b/demo-site/utils/constants.js index 66bbf24..b31ac59 100644 --- a/demo-site/utils/constants.js +++ b/demo-site/utils/constants.js @@ -4,7 +4,7 @@ const path = require('path'); const JSON_SPACING = 4; -const SERVER_CONFIG_FILE_NAME = 'config.json'; +const SERVER_CONFIG_FILE_NAME = 'px_config.json'; const SERVER_CONFIG_INC_FILE_NAME = "config.inc.json"; const PX_ENFORCER_CONFIG_FILE_NAME = 'px_config.json'; From 18f5a9f7cc2adff8d0e7ae44e9a0ce09690ba72b Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Sun, 6 Aug 2023 18:22:06 +0300 Subject: [PATCH 11/13] fix comments after pr --- demo-site/utils/constants.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/demo-site/utils/constants.js b/demo-site/utils/constants.js index b31ac59..fadd9c7 100644 --- a/demo-site/utils/constants.js +++ b/demo-site/utils/constants.js @@ -14,16 +14,8 @@ const SERVERS_DIRECTORY_PATH = path.join(__dirname, `../${SERVERS_DIRECTORY_NAME const FIRST_PARTY_STATIC_FILE_SUFFIX = ".firstparty"; const THIRD_PARTY_STATIC_FILE_SUFFIX = ""; -const CDN_DEPLOY_TOOL_CONFIG_FILE_NAME = "cliConfig.json"; -const CDN_DEPLOY_TOOL_RUN_CLI_COMMAND = "npm run cli"; -const CDN_DEPLOY_TOOL_BUILD_COMMAND = "npm run build"; - const PX_METADATA_FILE_NAME = "px_metadata.json"; -const TEST_APP_CREDENTIALS_ENDPOINT = "/test-app-credentials"; -const SUPPORTED_FEATURES_ENDPOINT = "/supported-features"; -const CONFIG_ENDPOINT = "/config"; - const INDEX_ROUTE = "/"; const LOGIN_ROUTE = "/login"; const LOGOUT_ROUTE = "/logout"; @@ -40,13 +32,7 @@ module.exports = { SERVERS_DIRECTORY_PATH, FIRST_PARTY_STATIC_FILE_SUFFIX, THIRD_PARTY_STATIC_FILE_SUFFIX, - CDN_DEPLOY_TOOL_CONFIG_FILE_NAME, - CDN_DEPLOY_TOOL_RUN_CLI_COMMAND, - CDN_DEPLOY_TOOL_BUILD_COMMAND, PX_METADATA_FILE_NAME, - TEST_APP_CREDENTIALS_ENDPOINT, - SUPPORTED_FEATURES_ENDPOINT, - CONFIG_ENDPOINT, INDEX_ROUTE, LOGIN_ROUTE, LOGOUT_ROUTE, From 918398c65d6922e5d02428b314fa9a772ca3c8d2 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Sun, 6 Aug 2023 18:26:55 +0300 Subject: [PATCH 12/13] fix comments after pr --- demo-site/utils/constants.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo-site/utils/constants.js b/demo-site/utils/constants.js index fadd9c7..0362b7f 100644 --- a/demo-site/utils/constants.js +++ b/demo-site/utils/constants.js @@ -7,6 +7,7 @@ const JSON_SPACING = 4; const SERVER_CONFIG_FILE_NAME = 'px_config.json'; const SERVER_CONFIG_INC_FILE_NAME = "config.inc.json"; const PX_ENFORCER_CONFIG_FILE_NAME = 'px_config.json'; +const CONFIG_FILE_NAME = 'px_config.json'; const SERVERS_DIRECTORY_NAME = "servers"; const SERVERS_DIRECTORY_PATH = path.join(__dirname, `../${SERVERS_DIRECTORY_NAME}`); @@ -25,6 +26,7 @@ const EXPECTED_PASSWORD = "1234"; module.exports = { JSON_SPACING, + CONFIG_FILE_NAME, SERVER_CONFIG_FILE_NAME, SERVER_CONFIG_INC_FILE_NAME, PX_ENFORCER_CONFIG_FILE_NAME, From 6aea47c390659d0f9e73c78beec1d83dd3f80673 Mon Sep 17 00:00:00 2001 From: Dan Bezalel Date: Sun, 6 Aug 2023 18:28:35 +0300 Subject: [PATCH 13/13] fix comments after pr --- demo-site/utils/constants.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/demo-site/utils/constants.js b/demo-site/utils/constants.js index 0362b7f..dd055c4 100644 --- a/demo-site/utils/constants.js +++ b/demo-site/utils/constants.js @@ -6,7 +6,6 @@ const JSON_SPACING = 4; const SERVER_CONFIG_FILE_NAME = 'px_config.json'; const SERVER_CONFIG_INC_FILE_NAME = "config.inc.json"; -const PX_ENFORCER_CONFIG_FILE_NAME = 'px_config.json'; const CONFIG_FILE_NAME = 'px_config.json'; const SERVERS_DIRECTORY_NAME = "servers"; @@ -29,7 +28,6 @@ module.exports = { CONFIG_FILE_NAME, SERVER_CONFIG_FILE_NAME, SERVER_CONFIG_INC_FILE_NAME, - PX_ENFORCER_CONFIG_FILE_NAME, SERVERS_DIRECTORY_NAME, SERVERS_DIRECTORY_PATH, FIRST_PARTY_STATIC_FILE_SUFFIX,