-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yaml
118 lines (116 loc) · 2.88 KB
/
docker-compose.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
version: "3"
services:
de_psql:
image: postgres:15
container_name: de_psql
volumes:
- ./postgresql:/var/lib/postgresql/data
ports:
- "5432:5432"
environment:
- POSTGRES_HOST=de_psql
- POSTGRES_PORT=5432
- POSTGRES_DB= postgres
- POSTGRES_USER=admin
- POSTGRES_PASSWORD=admin123
- POSTGRES_HOST_AUTH_METHOD=trust
networks:
iceberg_net:
# spark-iceberg:
# image: tabulario/spark-iceberg
# container_name: spark-iceberg
# build:
# context: spark/
# args:
# SPARK_DEFAULTS_CONF: ./spark-defaults-pg-catalog.conf
# networks:
# iceberg_net:
# depends_on:
# - rest
# - minio
# - de_psql
# volumes:
# - ./warehouse:/home/iceberg/warehouse
# - ./notebooks:/home/iceberg/notebooks/notebooks
# environment:
# - AWS_ACCESS_KEY_ID=admin
# - AWS_SECRET_ACCESS_KEY=password
# - AWS_REGION=us-east-1
# ports:
# - 8888:8888
# - 8080:8080
# - 10000:10000
# - 10001:10001
spark-iceberg:
build:
context: ./spark
args:
SPARK_DEFAULTS_CONF: spark-defaults-pg-catalog.conf
networks:
iceberg_net:
depends_on:
- rest
- minio
- de_psql
volumes:
- ./warehouse:/home/iceberg/warehouse
- ./notebooks:/home/iceberg/notebooks/notebooks
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
ports:
- 8888:8888
- 8080:8080
- 10000:10000
- 10001:10001
rest:
image: tabulario/iceberg-rest
container_name: iceberg-rest
networks:
iceberg_net:
ports:
- 8181:8181
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
- CATALOG_WAREHOUSE=s3://warehouse/
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
- CATALOG_S3_ENDPOINT=http://localhost:9000
minio:
image: minio/minio
container_name: minio
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
- MINIO_DOMAIN=minio
networks:
iceberg_net:
aliases:
- warehouse.minio
ports:
- 9001:9001
- 9000:9000
command: ["server", "/data", "--console-address", ":9001"]
mc:
depends_on:
- minio
image: minio/mc
container_name: mc
networks:
iceberg_net:
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=admin
- AWS_REGION=us-east-1
entrypoint: >
/bin/sh -c "
until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
/usr/bin/mc rm -r --force minio/warehouse;
/usr/bin/mc mb minio/warehouse;
/usr/bin/mc policy set public minio/warehouse;
tail -f /dev/null
"
networks:
iceberg_net: