docker-compose.yml

version: '3'

networks:
  app-tier:
    name: talk-streaming-network

services:
  zookeeper:
    image: 'bitnami/zookeeper:latest'
    container_name: zookeeper
    networks:
      - app-tier
    environment:
      - "ALLOW_ANONYMOUS_LOGIN=yes"
  kafka:
    image: confluentinc/cp-kafka:5.4.10
    hostname: kafka
    container_name: kafka
    networks:
      - app-tier
    depends_on:
      - zookeeper
    ports:
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1


  ksqldb-server:
    image: confluentinc/ksqldb-server:0.28.2
    hostname: ksqldb-server
    container_name: ksqldb-server
    depends_on:
      - kafka
    ports:
      - "8088:8088"
    networks:
      - app-tier
    environment:
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_BOOTSTRAP_SERVERS: kafka:9092
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
      KSQL_KSQL_CONNECT_WORKER_CONFIG: "/connect/connect.properties"
    volumes:
      - ./ksqldb/connect.properties:/connect/connect.properties

  ksqldb-cli:
    image: confluentinc/ksqldb-cli:0.28.2
    container_name: ksqldb-cli
    depends_on:
      - kafka
      - ksqldb-server
    networks:
      - app-tier
    entrypoint: /bin/sh
    tty: true
    volumes:
      - "./ksqldb-cli:/tmp/ksqldb"

  akhq:
    image: tchiotludo/akhq
    container_name: akhq
    networks:
      - app-tier    
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
          connections:
            docker-kafka-server:
              properties:
                bootstrap.servers: "kafka:9092"
    ports:
      - 8090:8080
    depends_on:
      - kafka

  datareference:
      image: postgres:14.3
      container_name: datareference
      ports:
        - "5432:5432"
      environment:
        - POSTGRES_PASSWORD=password
        - POSTGRES_USER=postgres
        - POSTGRES_DB=referencedata
      volumes:
        - "./postgresql/input_postgresql-docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/"
      networks:
        - app-tier

  nifi:
    image: apache/nifi:1.17.0
    container_name: nifi
    ports:
      - "8888:8080"
      - "8443:8443"
    volumes:
      - "./nifi/lib/:/usr/lib/postgres"
    environment:
      - "SINGLE_USER_CREDENTIALS_USERNAME=admin"
      - "SINGLE_USER_CREDENTIALS_PASSWORD=ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB"
    networks:
      - app-tier

  nifi-registry:
    image: apache/nifi-registry:1.17.0
    container_name: nifi-registry
    ports:
      - "18080:18080"
    networks:
      - app-tier
    volumes:
      - ./nifiRegistry/flow-storage:/opt/nifi-registry/nifi-registry-current/flow_storage
      - ./nifiRegistry/database:/opt/nifi-registry/nifi-registry-current/database

  flink-jobmanager:
    build:
      context: ./flink
    ports:
      - "8085:8081"
    command: jobmanager
    container_name: flink-jobmanager
    networks:
      - app-tier    
    volumes:
      - "./flink/stream_java_api/target:/usr/local/flink/stream_java_api"
      - ./flink/table_api:/usr/local/flink/table_api
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager        

  flink-taskmanager:
    build:
      context: ./flink
    depends_on:
      - flink-jobmanager
    command: taskmanager
    container_name: flink-taskmanager
    scale: 1
    networks:
      - app-tier
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 10

 # kafka-streams:
 #   image: adoptopenjdk/openjdk11
 #   depends_on:
 #     - kafka
 #   container_name: kafkaStreams
 #   volumes: 
 #     - "./kafkaStreams/target:/tmp"
 #   command: java -jar /tmp/kafka-stream-demo-1.0-SNAPSHOT.jar
 #   networks:
 #     - app-tier
    
  influxdb:
    image: influxdb:2.1.0-alpine
    container_name: influxdb
    networks:
      - app-tier
    environment:
      DOCKER_INFLUXDB_INIT_MODE: setup
      DOCKER_INFLUXDB_INIT_USERNAME: volcamp
      DOCKER_INFLUXDB_INIT_PASSWORD: this_is_my_long_password
      DOCKER_INFLUXDB_INIT_ORG: volcamp
      DOCKER_INFLUXDB_INIT_BUCKET: volcamp
      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: LbL3syu4EXiQ5Z8Fkscs6TbLq7jC8gGo2HdRqH1vM32llA_1xWaVwIAzPHKoERGynxIEzsWMREQPmTDIcfYskg==
    ports:
      - "8086:8086"

  telegraf:
    image: telegraf:1.21.1-alpine
    container_name: telegraf
    networks:
      - app-tier
    environment:
      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: LbL3syu4EXiQ5Z8Fkscs6TbLq7jC8gGo2HdRqH1vM32llA_1xWaVwIAzPHKoERGynxIEzsWMREQPmTDIcfYskg==
      DOCKER_INFLUXDB_INIT_ORG: volcamp
      DOCKER_INFLUXDB_INIT_BUCKET: volcamp
    volumes:
      - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf
    depends_on:
      - influxdb
      - kafka
      
# control-center:
#   image: confluentinc/cp-enterprise-control-center:5.4.10
#   container_name: control-center
#   networks:
#     - app-tier
#   depends_on:
#     - zookeeper
#     - kafka
#     - ksqldb-server
#   ports:
#     - "9021:9021"
#   environment:
#     CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:9092'
#     CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181'
#     CONTROL_CENTER_KSQL_URL: "http://ksqldb-server:8088"
#     CONTROL_CENTER_KSQL_ADVERTISED_URL: "http://ksqldb-server:8088"
#     CONFLUENT_METRICS_TOPIC_REPLICATION: 1
#     CONTROL_CENTER_REPLICATION_FACTOR: 1
#     CONTROL_CENTER_COMMAND_TOPIC_REPLICATION: 1
#     CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_REPLICATION: 1
#     CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
#     CONTROL_CENTER_INTERNAL_TOPICS_REPLICATION: 1
#     CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
#     CONTROL_CENTER_STREAMS_NUM_STREAM_THREADS: 1
#     CONTROL_CENTER_STREAMS_CACHE_MAX_BYTES_BUFFERING: 104857600
#     CONTROL_CENTER_DEPRECATED_VIEWS_ENABLE: "true"