From 01648d5d3cea168adee3f37679edd7c2a364c681 Mon Sep 17 00:00:00 2001 From: shibukazu Date: Sat, 24 Aug 2024 05:07:18 +0900 Subject: [PATCH] add readme --- README.md | 203 ++++++++--------------------------- docs/Master-Slave-Example.md | 181 +++++++++++++++++++++++++++++++ docs/Monolithic-Example.md | 133 +++++++++++++++++++++++ 3 files changed, 358 insertions(+), 159 deletions(-) create mode 100644 docs/Master-Slave-Example.md create mode 100644 docs/Monolithic-Example.md diff --git a/README.md b/README.md index b0da5f7..6e217d6 100644 --- a/README.md +++ b/README.md @@ -18,24 +18,29 @@ Open-VE offers an HTTP API and a gRPC API. We will provide a client SDK in the f #### 1. CLI Flags or Environment Variables -| CLI Args | Env | Default | Desc | -| ----------------------------- | ----------------------------------- | ------------ | --------------------------- | -| `--http-addr` | `OPEN-VE_HTTP_ADDR` | `:8080` | HTTP server address | -| `--http-cors-allowed-origins` | `OPEN-VE_HTTP_CORS_ALLOWED_ORIGINS` | `["*"]` | CORS allowed origins | -| `--http-cors-allowed-headers` | `OPEN-VE_HTTP_CORS_ALLOWED_HEADERS` | `["*"]` | CORS allowed headers | -| `--http-tls-enabled` | `OPEN-VE_HTTP_TLS_ENABLED` | `false` | HTTP server TLS enabled | -| `--http-tls-cert-path` | `OPEN-VE_HTTP_TLS_CERT_PATH` | `""` | HTTP server TLS cert path | -| `--http-tls-key-path` | `OPEN-VE_HTTP_TLS_KEY_PATH` | `""` | HTTP server TLS key path | -| `--grpc-addr` | `OPEN-VE_GRPC_ADDR` | `:9000` | gRPC server address | -| `--grpc-tls-enabled` | `OPEN-VE_GRPC_TLS_ENABLED` | `false` | gRPC server TLS enabled | -| `--grpc-tls-cert-path` | `OPEN-VE_GRPC_TLS_CERT_PATH` | `""` | gRPC server TLS cert path | -| `--grpc-tls-key-path` | `OPEN-VE_GRPC_TLS_KEY_PATH` | `""` | gRPC server TLS key path | -| `--store-engine` | `OPEN-VE_STORE_ENGINE` | `memory` | store engine (redis/memory) | -| `--store-redis-addr` | `OPEN-VE_STORE_REDIS_ADDR` | `redis:6379` | Redis address | -| `--store-redis-password` | `OPEN-VE_STORE_REDIS_PASSWORD` | `""` | Redis password | -| `--store-redis-db` | `OPEN-VE_STORE_REDIS_DB` | `0` | Redis DB | -| `--store-redis-pool-size` | `OPEN-VE_STORE_REDIS_POOL_SIZE` | `1000` | Redis pool size | -| `--log-level` | `OPEN-VE_LOG_LEVEL` | `info` | Log level | +| CLI Args | Env | Default | Desc | +| ----------------------------- | ----------------------------------- | ------------ | ---------------------------------------------------------------------- | +| `--mode` | `OPEN-VE_MODE` | `master` | master or slave | +| `--slave-id` | `OPEN-VE_SLAVE_ID` | | Unique slave ID (if mode is slave, this is required) | +| `--slave-slave-http-addr` | `OPEN-VE_SLAVE_SLAVE_HTTP_ADDR` | | HTTP server address (if mode is slave, this is required) | +| `--slave-master-http-addr` | `OPEN-VE_SLAVE_MASTER_HTTP_ADDR` | | Master HTTP server address (if mode is slave, this is required) | +| `--slave-master-tsl-enabled` | `OPEN-VE_SLAVE_MASTER_TLS_ENABLED` | `false` | Connect to master server with TLS (if mode is slave, this is required) | +| `--http-port` | `OPEN-VE_HTTP_PORT` | `8080` | HTTP server port number | +| `--http-cors-allowed-origins` | `OPEN-VE_HTTP_CORS_ALLOWED_ORIGINS` | `["*"]` | CORS allowed origins | +| `--http-cors-allowed-headers` | `OPEN-VE_HTTP_CORS_ALLOWED_HEADERS` | `["*"]` | CORS allowed headers | +| `--http-tls-enabled` | `OPEN-VE_HTTP_TLS_ENABLED` | `false` | HTTP server TLS enabled | +| `--http-tls-cert-path` | `OPEN-VE_HTTP_TLS_CERT_PATH` | `""` | HTTP server TLS cert path | +| `--http-tls-key-path` | `OPEN-VE_HTTP_TLS_KEY_PATH` | `""` | HTTP server TLS key path | +| `--grpc-port` | `OPEN-VE_GRPC_ADDR` | `:9000` | gRPC server port number | +| `--grpc-tls-enabled` | `OPEN-VE_GRPC_TLS_ENABLED` | `false` | gRPC server TLS enabled | +| `--grpc-tls-cert-path` | `OPEN-VE_GRPC_TLS_CERT_PATH` | `""` | gRPC server TLS cert path | +| `--grpc-tls-key-path` | `OPEN-VE_GRPC_TLS_KEY_PATH` | `""` | gRPC server TLS key path | +| `--store-engine` | `OPEN-VE_STORE_ENGINE` | `memory` | store engine (redis/memory) | +| `--store-redis-addr` | `OPEN-VE_STORE_REDIS_ADDR` | `redis:6379` | Redis address | +| `--store-redis-password` | `OPEN-VE_STORE_REDIS_PASSWORD` | `""` | Redis password | +| `--store-redis-db` | `OPEN-VE_STORE_REDIS_DB` | `0` | Redis DB | +| `--store-redis-pool-size` | `OPEN-VE_STORE_REDIS_POOL_SIZE` | `1000` | Redis pool size | +| `--log-level` | `OPEN-VE_LOG_LEVEL` | `info` | Log level | #### 2. Config File @@ -44,8 +49,9 @@ You can also use a config file in YAML format. Place the `config.yaml` in the same directory or `$HOME/.open-ve/config.yaml`. ```yaml +mode: "master" http: - addr: ":8080" + port: "8080" corsAllowedOrigins: ["*"] corsAllowedHeaders: ["*"] tls: @@ -53,7 +59,7 @@ http: certPath: "" keyPath: "" grpc: - addr: ":9000" + poer: "9000" tls: enabled: false certPath: "" @@ -91,7 +97,21 @@ go build -o open-ve go/cmd/open-ve/main.go docker-compose up ``` -## CEL +## System Design + +### Master-Slave Architecture + +Open-VE supports a master-slave architecture designed for scalability and compatibility with microservice environments. + +In slave mode, Open-VE connects to the master server and syncs validation rules every 30 seconds. + +When a validation check request is made to the master server, it distributes the request across the connected slave servers. + +Additionally, you can directly request validation checks from the slave servers. + +![micro-validator](https://github.com/user-attachments/assets/e248d40c-bcc7-4219-a65a-5b243e101000) + +### CEL We use [CEL](https://github.com/google/cel-spec/blob/master/doc/langdef.md) as the expression language for validation rules. @@ -111,143 +131,8 @@ Supported types: | message names | | ❓ | | `type` | | ❓ | -## Example (HTTP API) - -### Register Validation Rules - -Request: - -```bash -curl --request POST \ - --url http://localhost:8080/v1/dsl \ - --header 'Content-Type: application/json' \ - --data '{ - "validations": [ - { - "id": "item", - "cels": [ - "price > 0", # price must be greater than 0 - "size(image) < 360" # image size must be less than 360 bytes - ], - "variables": [ - { - "name": "price", - "type": "int" - }, - { - "name": "image", - "type": "bytes" - } - ] - }, - { - "id": "user", - "cels": [ - "size(name) < 20" # name length must be less than 20 - ], - "variables": [ - { - "name": "name", - "type": "string" - } - ] - } - ] - }' -``` - -Response: - -```json -{} -``` - -### Get Current Validation Rules - -Request: - -```bash -curl --request GET \ - --url http://localhost:8080/v1/dsl \ - --header 'Content-Type: application/json' -``` - -Response: - -```json -{ - "validations": [ - { - "id": "item", - "cels": ["price > 0", "size(image) < 360"], - "variables": [ - { - "name": "price", - "type": "int" - }, - { - "name": "image", - "type": "bytes" - } - ] - }, - { - "id": "user", - "cels": ["size(name) < 20"], - "variables": [ - { - "name": "name", - "type": "string" - } - ] - } - ] -} -``` - -### Validate +## Examples -Request: - -```bash -curl --request POST \ - --url 'http://localhost:8080/v1/check' \ - --header 'Content-Type: application/json' \ - --data '{ - "validations": [ - { - "id": "item", - "variables": { - "price": -100, - "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGO4unY2AAR4Ah51j5XwAAAAAElFTkSuQmCC" # send base64 encoded image - } - }, - { - "id": "user", - "variables": { - "name": "longlonglonglongname" - } - } - ] - }' +- [Example of Master Slave Architecture](docs/Master-Slave-Example.md) -``` - -Response: - -```json -{ - "results": [ - { - "id": "item", - "isValid": false, - "message": "failed validations: price > 0" - }, - { - "id": "user", - "isValid": false, - "message": "failed validations: size(name) < 20" - } - ] -} -``` +- [Example of Monolithic Architecture](docs/Monolithic-Example.md) diff --git a/docs/Master-Slave-Example.md b/docs/Master-Slave-Example.md new file mode 100644 index 0000000..5a7a889 --- /dev/null +++ b/docs/Master-Slave-Example.md @@ -0,0 +1,181 @@ +# Master-SlaveExample + +This example demonstrates how to run the Open-VE as master-slave architecture. + +Note: In this example, the master node is hosted at `localhost:8081`, and the slave node is hosted at `localhost:8082`. + +## Run + +### Master Node + +```bash +OPEN-VE_MODE=master +OPEN-VE_HTTP_PORT=8081 +OPEN-VE_GRPC_PORT=9001 + +open-ve run +``` + +### Slave Node + +```bash +OPEN-VE_MODE=slave +OPEN-VE_SLAVE_ID=slave-node-id +OPEN-VE_SLAVE_MASTER_HTTP_ADDR=http://localhost:8081 +OPEN-VE_SLAVE_SLAVE_HTTP_ADDR=http://localhost:8082 +OPEN-VE_HTTP_PORT=8082 +OPEN-VE_GRPC_PORT=9002 + +open-ve run +``` + +## Scinario + +### 1. Register a Set of Validation Rules to Master Node + +```bash +curl --request POST \ + --url http://localhost:8081/v1/dsl \ + --header 'Content-Type: application/json' \ + --data '{ + "validations": [ + { + "id": "user", + "cels": [ + "size(name) < 20" + ], + "variables": [ + { + "name": "name", + "type": "string" + } + ] + } + ] +}' +``` + +### 2. Register a Set of Validation Rules to Slave Node + +```bash +curl --request POST \ + --url http://localhost:8082/v1/dsl \ + --header 'Content-Type: application/json' \ + --data '{ + "validations": [ + { + "id": "item", + "cels": [ + "price > 0", + "size(image) < 360" + ], + "variables": [ + { + "name": "price", + "type": "int" + }, + { + "name": "image", + "type": "bytes" + } + ] + } + ] +}' +``` + +### 3. Check the Current Validation Rules + +```bash +curl --request GET \ + --url http://localhost:8081/v1/dsl \ + --header 'Content-Type: application/json' +``` + +```json +{ + "validations": [ + { + "id": "user", + "cels": ["size(name) < 20"], + "variables": [ + { + "name": "name", + "type": "string" + } + ] + } + ] +} +``` + +```bash +curl --request GET \ + --url http://localhost:8082/v1/dsl \ + --header 'Content-Type: application/json' +``` + +```json +{ + "validations": [ + { + "id": "item", + "cels": ["price > 0", "size(image) < 360"], + "variables": [ + { + "name": "price", + "type": "int" + }, + { + "name": "image", + "type": "bytes" + } + ] + } + ] +} +``` + +### 4. Request Validation Check to Master Node + +Although only part of the validation rules are registered with the master node, you can request validation for all rules, including those on the slave nodes. + +```bash +curl --request POST \ + --url http://localhost:8081/v1/check \ + --header 'Content-Type: application/json' \ + --data '{ + "validations": [ + { + "id": "item", + "variables": { + "price": -100, + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGO4unY2AAR4Ah51j5XwAAAAAElFTkSuQmCC" + } + }, + { + "id": "user", + "variables": { + "name": "longlonglonglongname" + } + } + ] +}' +``` + +```json +{ + "results": [ + { + "id": "user", + "isValid": false, + "message": "failed validations: size(name) < 20" + }, + { + "id": "item", + "isValid": false, + "message": "failed validations: price > 0" + } + ] +} +``` diff --git a/docs/Monolithic-Example.md b/docs/Monolithic-Example.md new file mode 100644 index 0000000..cf75e84 --- /dev/null +++ b/docs/Monolithic-Example.md @@ -0,0 +1,133 @@ +# Monolithic Example + +This example demonstrates how to run the Open-VE as monolithic architecture. + +## Run + +```bash +open-ve run +``` + +## Scenario + +### 1. Register Validation Rules + +```bash +curl --request POST \ + --url http://localhost:8080/v1/dsl \ + --header 'Content-Type: application/json' \ + --data '{ + "validations": [ + { + "id": "user", + "cels": [ + "size(name) < 20" + ], + "variables": [ + { + "name": "name", + "type": "string" + } + ] + } + { + "id": "item", + "cels": [ + "price > 0", + "size(image) < 360" + ], + "variables": [ + { + "name": "price", + "type": "int" + }, + { + "name": "image", + "type": "bytes" + } + ] + } + ] +}' +``` + +### 2. Check the Current Validation Rules + +```bash +curl --request GET \ + --url http://localhost:8080/v1/dsl \ + --header 'Content-Type: application/json' +``` + +```json +{ + "validations": [ + { + "id": "user", + "cels": ["size(name) < 20"], + "variables": [ + { + "name": "name", + "type": "string" + } + ] + } + { + "id": "item", + "cels": ["price > 0", "size(image) < 360"], + "variables": [ + { + "name": "price", + "type": "int" + }, + { + "name": "image", + "type": "bytes" + } + ] + } + ] +} +``` + +### 3. Request Validation Check + +```bash +curl --request POST \ + --url http://localhost:8080/v1/check \ + --header 'Content-Type: application/json' \ + --data '{ + "validations": [ + { + "id": "item", + "variables": { + "price": -100, + "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGO4unY2AAR4Ah51j5XwAAAAAElFTkSuQmCC" + } + }, + { + "id": "user", + "variables": { + "name": "longlonglonglongname" + } + } + ] +}' +``` + +```json +{ + "results": [ + { + "id": "user", + "isValid": false, + "message": "failed validations: size(name) < 20" + }, + { + "id": "item", + "isValid": false, + "message": "failed validations: price > 0" + } + ] +} +```