-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
299 lines (284 loc) · 10.6 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
---
version: "3.2"
services:
zookeeper:
image: confluentinc/cp-zookeeper:5.4.1
hostname: zookeeper
container_name: zookeeper
# ports 2181, 2888, and 3888 are exposed by default
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
restart: unless-stopped
volumes:
- ./data/zookeeper/data:/var/lib/zookeeper/data
- ./data/zookeeper/log:/var/lib/zookeeper/log
networks:
- venice
broker-1:
image: confluentinc/cp-kafka:5.4.1
hostname: broker-1
container_name: broker-1
restart: unless-stopped
depends_on:
- zookeeper
# port 9092 is exposed by default
environment:
# parameters here:
# https://kafka.apache.org/documentation/#configuration
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
# KAFKA_ADVERTISED_LISTENERS is a comma-separated list of listeners with their
# host/IP and port. This is the metadata that’s passed back to clients.
# https://www.confluent.io/blog/kafka-listeners-explained/
# the use of PLAINTEXT is related to the security.inter.broker.protocol:
# https://docs.confluent.io/current/installation/configuration/broker-configs.html
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_LOG_RETENTION_BYTES: $LOG_RETENTION_BYTES
KAFKA_LOG_RETENTION_HOURS: $LOG_RETENTION_HOURS
volumes:
- ./data/broker-1/data:/var/lib/kafka/data
networks:
- venice
broker-2:
image: confluentinc/cp-kafka:5.4.1
hostname: broker-2
container_name: broker-2
restart: unless-stopped
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_LOG_RETENTION_BYTES: $LOG_RETENTION_BYTES
KAFKA_LOG_RETENTION_HOURS: $LOG_RETENTION_HOURS
networks:
- venice
volumes:
- ./data/broker-2/data:/var/lib/kafka/data
broker-3:
image: confluentinc/cp-kafka:5.4.1
hostname: broker-3
container_name: broker-3
restart: unless-stopped
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
KAFKA_LOG_RETENTION_BYTES: $LOG_RETENTION_BYTES
KAFKA_LOG_RETENTION_HOURS: $LOG_RETENTION_HOURS
networks:
- venice
volumes:
- ./data/broker-3/data:/var/lib/kafka/data
schema-registry:
image: confluentinc/cp-schema-registry:5.4.1
hostname: schema-registry
container_name: schema-registry
depends_on:
- broker-1
- broker-2
- broker-3
# port 8081 is exposed by default
# publish port to work with venice CLI
ports:
- 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
# zookeeper election is deprecated, use kafka instead
# https://docs.confluent.io/current/schema-registry/installation/config.html#kafkastore-bootstrap-servers
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://broker-1:9092,PLAINTEXT://broker-2:9092,PLAINTEXT://broker-3:9092
# Comma-separated list of listeners that listen for API requests over
# either HTTP or HTTPS. If a listener uses HTTPS, the appropriate SSL
# configuration parameters need to be set as well.
SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081"
DEBUG: "true"
networks:
- venice
producer:
image: veniceframework/python-producer-test-key-string
container_name: producer
depends_on:
- broker-1
- broker-2
- broker-3
- schema-registry
environment:
BROKER: "broker-1:9092"
SCHEMA_REGISTRY_HOST: "schema-registry"
SCHEMA_REGISTRY_PORT: 8081
SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
TOPIC_NAME: "$TOPIC_NAME"
networks:
- venice
ksql-server:
image: confluentinc/cp-ksql-server:5.4.1
hostname: ksql-server
container_name: ksql-server
depends_on:
- broker-1
- broker-2
- broker-3
- schema-registry
# no ports are exposed by default
ports:
- 8088:8088
environment:
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092
KSQL_KSQL_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
networks:
- venice
kafka-connect:
image: confluentinc/cp-kafka-connect:5.4.1
hostname: kafka-connect
container_name: kafka-connect
depends_on:
- broker-1
- broker-2
- broker-3
- schema-registry
# exposes ports 8083 and 9092 by default
# publish port to work with venice CLI
ports:
- 8083:8083
environment:
# Required Kafka Connect Settings:
# https://docs.confluent.io/current/installation/docker/config-reference.html#required-kafka-connect-settings
#
# All Settings:
# https://docs.confluent.io/current/connect/references/allconfigs.html
#
###### REQUIRED SETTINGS ######
CONNECT_BOOTSTRAP_SERVERS: broker-1:9092,broker-2:9092,broker-3:9092
CONNECT_GROUP_ID: venice
# The values for these three storage topics must be the same for all
# workers with the same group.id
CONNECT_CONFIG_STORAGE_TOPIC: "connect-1-config"
CONNECT_OFFSET_STORAGE_TOPIC: "connect-1-offsets"
CONNECT_STATUS_STORAGE_TOPIC: "connect-1-status"
# These control the format of the data that will be written to Kafka for
# source connectors or read from Kafka for sink connectors
CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
# Converter class for internal values that implements the Converter interface
# TODO: verify what these do
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
###### END REQUIRED SETTINGS ######
# This is the default port
CONNECT_REST_PORT: 8083
# These were not listed in the all settings page, but used in the demo above and tutorials
# https://docs.confluent.io/5.0.0/installation/docker/docs/installation/connect-avro-jdbc.html#starting-up-confluent-platform-and-kafka-connect
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
# These should always be at least 3 for a production system
# The defaults are 3
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3
networks:
- venice
connector-init:
image: alpine:3.11
container_name: connector-init
depends_on:
- kafka-connect
- postgres
command:
- sh
- -c
- |
apk add curl
echo "Waiting for kafka-connect to start listening"
while [ $$(curl -s -o /dev/null -w %{http_code} http://kafka-connect:8083/connectors) -eq 000 ] ; do
echo -e $$(date) " Kafka Connect listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://kafka-connect:8083/connectors) " (waiting for 200)"
sleep 5
done
echo "kafka-connect is ready"
nc -vz kafka-connect 8083
echo "Sleeping for 2 mins (TODO: https://github.com/venice-framework/venice-python/issues/12)"
sleep 2m
echo -e "\n--\n+> Creating postgres sink"
echo \
"{
\"name\" : \"postgres_${POSTGRES_DB}_$TOPIC_NAME\",
\"config\" : {
\"connector.class\" : \"io.confluent.connect.jdbc.JdbcSinkConnector\",
\"connection.url\": \"jdbc:postgresql://$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB\",
\"connection.user\": \"$POSTGRES_USER\",
\"connection.password\": \"$POSTGRES_PASSWORD\",
\"topics\": \"$TOPIC_NAME\",
\"value.converter\": \"io.confluent.connect.avro.AvroConverter\",
\"value.converter.schema.registry.url\": \"http://schema-registry:8081\",
\"key.converter\": \"org.apache.kafka.connect.storage.StringConverter\",
\"key.converter.schema.registry.url\": \"http://schema-registry:8081\",
\"auto.create\":\"true\",
\"auto.evolve\":\"true\",
\"insert.mode\": \"upsert\",
\"pk.mode\": \"record_key\",
\"pk.fields\": \"ROWKEY\"
}
}" | curl -s -X POST -d @- http://kafka-connect:8083/connectors -H "Content-Type: application/json" -w "\n"
networks:
- venice
postgres:
# https://hub.docker.com/_/postgres
# docker exec -it postgres psql --username=venice_user --dbname=MY_DB
build: ./postgres
container_name: postgres
# exposes port 5432 by default
expose:
- $POSTGRES_PORT
environment:
POSTGRES_USER: $POSTGRES_USER
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
POSTGRES_DB: $POSTGRES_DB
restart: unless-stopped
volumes:
- ./data/postgres:/var/lib/postgresql/data
networks:
- venice
kafka-ui:
# https://hub.docker.com/r/obsidiandynamics/kafdrop
image: obsidiandynamics/kafdrop:3.24.0
container_name: kafka-ui
ports:
- 9000:9000
depends_on:
- broker-1
- broker-2
- broker-3
environment:
KAFKA_BROKERCONNECT: broker-1:9092,broker-2:9092,broker-3:9092
JVM_OPTS: "-Xms32M -Xmx64M"
SERVER_PORT: 9000
SERVER_SERVLET_CONTEXTPATH: "/"
MESSAGE_FORMAT: AVRO
SCHEMAREGISTRY_CONNECT: http://schema-registry:8081
#JMX_PORT:
#HOST:
networks:
- venice
networks:
venice:
driver: bridge # bridge is the default
# licensing info for confluent images: https://docs.confluent.io/current/installation/docker/image-reference.html