From 71131f33ebb382e541200502ba489dd994676d05 Mon Sep 17 00:00:00 2001 From: Sagar Lakshmipathy <18vidhyasagar@gmail.com> Date: Sat, 25 Nov 2023 09:50:09 -0800 Subject: [PATCH 1/2] addresses 257 --- demo/Dockerfile | 12 +++++++++ demo/docker-compose.yaml | 53 +++++++++++++++++++++++++++++++--------- demo/start_demo.sh | 5 ++-- 3 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 demo/Dockerfile diff --git a/demo/Dockerfile b/demo/Dockerfile new file mode 100644 index 00000000..7faab2b7 --- /dev/null +++ b/demo/Dockerfile @@ -0,0 +1,12 @@ +FROM bitnami/spark:3.2.4 + +USER root + +# Install vim and cd to onetable +RUN apt-get update \ + && apt-get install -y vim \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /home/onetable \ + && chmod -R 777 /home/onetable + +WORKDIR /home/onetable \ No newline at end of file diff --git a/demo/docker-compose.yaml b/demo/docker-compose.yaml index 0ff00f42..4e31d88a 100644 --- a/demo/docker-compose.yaml +++ b/demo/docker-compose.yaml @@ -1,5 +1,46 @@ version: "3.9" services: + spark-master: + build: + context: . + dockerfile: Dockerfile # Dockerfile creates /home/onetable dir and gives permissions to user + environment: + - SPARK_MODE=master + - SPARK_MASTER_PORT=7077 + ports: + - "18080:18080" # Spark master web UI port + - "7077:7077" # Spark master port + volumes: + - ./jars/utilities-0.1.0-SNAPSHOT-bundled.jar:/home/onetable/utilities/target/utilities-0.1.0-SNAPSHOT-bundled.jar + + spark-worker-1: + image: bitnami/spark:3.2.4 + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + depends_on: + - spark-master + + spark-worker-2: + image: bitnami/spark:3.2.4 + environment: + - SPARK_MODE=worker + - SPARK_MASTER_URL=spark://spark-master:7077 + depends_on: + - spark-master + + hive-metastore: + container_name: hive-metastore + hostname: hive-metastore + image: 'apache/hive:4.0.0-alpha-2' + ports: + - '9083:9083' # Metastore Thrift + environment: + SERVICE_NAME: metastore + HIVE_METASTORE_WAREHOUSE_DIR: /home/data + volumes: + - ./data:/home/data + trino: container_name: trino ports: @@ -21,18 +62,6 @@ services: - ./presto/node.properties:/opt/presto-server/etc/node.properties - ./data:/home/data - hive-metastore: - container_name: hive-metastore - hostname: hive-metastore - image: 'apache/hive:4.0.0-alpha-2' - ports: - - '9083:9083' # Metastore Thrift - environment: - SERVICE_NAME: metastore - HIVE_METASTORE_WAREHOUSE_DIR: /home/data - volumes: - - ./data:/home/data - jupyter: container_name: jupyter hostname: jupyter diff --git a/demo/start_demo.sh b/demo/start_demo.sh index 253b405c..d6b577ee 100755 --- a/demo/start_demo.sh +++ b/demo/start_demo.sh @@ -1,10 +1,11 @@ #!/bin/bash -# Create the require jars for the demo and copy them into a directory we'll mount in our notebook container -cd .. && mvn install -am -pl core -DskipTests -T 2 +# Create the require jars for the demo and copy them into a directory we'll mount in our notebook & spark container +cd .. && mvn install -DskipTests -T 2 mkdir -p demo/jars cp hudi-support/utils/target/hudi-utils-0.1.0-SNAPSHOT.jar demo/jars cp api/target/onetable-api-0.1.0-SNAPSHOT.jar demo/jars cp core/target/onetable-core-0.1.0-SNAPSHOT.jar demo/jars +cp utilities/target/utilities-0.1.0-SNAPSHOT-bundled.jar demo/jars cd demo docker-compose up \ No newline at end of file From 44a1e2e0ac37da096724d6bfae6c1ed87d7e1a16 Mon Sep 17 00:00:00 2001 From: Sagar Lakshmipathy <18vidhyasagar@gmail.com> Date: Sat, 25 Nov 2023 11:47:04 -0800 Subject: [PATCH 2/2] added and clarified docs --- .../docs/{demo/docker.md => docker/demo.md} | 0 website/docs/docker/playground.md | 36 +++++++++++++++++++ website/sidebars.js | 14 ++++++-- 3 files changed, 47 insertions(+), 3 deletions(-) rename website/docs/{demo/docker.md => docker/demo.md} (100%) create mode 100644 website/docs/docker/playground.md diff --git a/website/docs/demo/docker.md b/website/docs/docker/demo.md similarity index 100% rename from website/docs/demo/docker.md rename to website/docs/docker/demo.md diff --git a/website/docs/docker/playground.md b/website/docs/docker/playground.md new file mode 100644 index 00000000..d7920dba --- /dev/null +++ b/website/docs/docker/playground.md @@ -0,0 +1,36 @@ +--- +sidebar_position: 2 +title: "Docker Playground" +--- + +# OneTable Playground +This playground helps you install and work with OneTable in a dockerized environment. + +## Pre-requisites +* Install Docker in your local machine +* Clone [OneTable GitHub repository](https://github.com/onetable-io/onetable) + +:::note NOTE: +This demo was tested in AArch64 based macOS operating system +::: + +## Setting up the playground +After cloning the OneTable repository, change directory to `demo` and run the `start_demo.sh` script. +This script builds OneTable jars required for the demo and then spins up docker containers to start a Spark cluster, +Jupyter notebook with Scala interpreter, Hive Metastore, Presto and Trino. + +```shell md title="shell" +cd demo +./start_demo.sh +``` + +### Accessing the Spark cluster +You can access the Spark master by running `docker exec -it demo-spark-master-1 /bin/bash` in a separate terminal window. +This will open the master node in a bash shell and will initially take you to `/home/onetable` directory. + +### Running the tutorial +Once inside the master node, you can follow the [Creating your first interoperable table](/docs/how-to#steps) tutorial. + +To run sync, from the same directory, you can create the my_config.yaml file using `vi my_config.yaml` command and copy-paste +the contents from the [run sync](/docs/how-to#running-sync) section and then +run the `java -jar utilities/target/utilities-0.1.0-SNAPSHOT-bundled.jar --datasetConfig my_config.yaml` command. diff --git a/website/sidebars.js b/website/sidebars.js index b8743d96..d10312e5 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -29,7 +29,7 @@ module.exports = { { type: 'category', label: 'Catalogs', - collapsed: false, + collapsed: true, link: { type: 'doc', id: 'catalogs-index' @@ -44,7 +44,7 @@ module.exports = { { type: 'category', label: 'Query Engines', - collapsed: false, + collapsed: true, link: { type: 'doc', id: 'query-engines-index' @@ -62,6 +62,14 @@ module.exports = { } ] }, - 'demo/docker', + { + type: 'category', + label: 'Docker', + collapsed: false, + items: [ + 'docker/demo', + 'docker/playground', + ], + } ], }; \ No newline at end of file