################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

version: "3"

services:

  # ----------------------------------------
  # Flink services
  # ----------------------------------------

  jobmanager:
    image: apache/flink:${test.flink.version}-${test.java.version}
    volumes:
      - testdata:/test-data
      - /tmp/paimon-e2e-tests-jars:/jars
    entrypoint: >
      /bin/bash -c "
      cp /jars/paimon-flink.jar /jars/paimon-flink-action.jar /jars/bundled-hadoop.jar 
      /jars/mysql-cdc.jar /jars/mysql-connector-java.jar
      /jars/flink-sql-connector-kafka.jar /jars/flink-sql-connector-hive.jar /opt/flink/lib ;
      echo 'See FLINK-31659 for why we need the following two steps' ;
      mv /opt/flink/opt/flink-table-planner*.jar /opt/flink/lib/ ;
      mv /opt/flink/lib/flink-table-planner-loader-*.jar /opt/flink/opt/ ;
      /docker-entrypoint.sh jobmanager
      "
    env_file:
      - ./flink.env
    networks:
      testnetwork:
        aliases:
          - jobmanager
    expose:
      - "8081"

  taskmanager:
    image: apache/flink:${test.flink.version}-${test.java.version}
    volumes:
      - testdata:/test-data
      - /tmp/paimon-e2e-tests-jars:/jars
    entrypoint: >
      /bin/bash -c "
      cp /jars/paimon-flink.jar /jars/paimon-flink-action.jar /jars/bundled-hadoop.jar 
      /jars/mysql-cdc.jar /jars/mysql-connector-java.jar
      /jars/flink-sql-connector-kafka.jar /jars/flink-sql-connector-hive.jar /opt/flink/lib ;
      echo 'See FLINK-31659 for why we need the following two steps' ;
      mv /opt/flink/opt/flink-table-planner*.jar /opt/flink/lib/ ;
      mv /opt/flink/lib/flink-table-planner-loader-*.jar /opt/flink/opt/ ;
      /docker-entrypoint.sh taskmanager
      "
    env_file:
      - ./flink.env
    networks:
      testnetwork:
        aliases:
          - taskmanager
    depends_on:
      - jobmanager

  # ----------------------------------------
  # Kafka services
  # ----------------------------------------

  kafka:
    image: confluentinc/cp-kafka:7.8.0
    networks:
      testnetwork:
        aliases:
          - kafka
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
      KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_TRANSACTION_MAX_TIMEOUT_MS: 7200000
      # Disable log deletion to prevent records from being deleted during test run
      KAFKA_LOG_RETENTION_MS: -1
      CLUSTER_ID: Pkv3OFSRNQcNATKEMDV9Ql
    expose:
      - "9092"

  # ----------------------------------------
  # Hive services, copied and modified from https://github.com/big-data-europe/docker-hive
  # ----------------------------------------

  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8
    volumes:
      - testdata:/test-data
      - namenode:/hadoop/dfs/name
    networks:
      testnetwork:
        aliases:
          - namenode
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - "50070"

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
    volumes:
      - testdata:/test-data
      - datanode:/hadoop/dfs/data
    networks:
      testnetwork:
        aliases:
          - datanode
    env_file:
      - ./hadoop-hive.env
    environment:
      SERVICE_PRECONDITION: "namenode:50070"
    ports:
      - "50075"

  hive-server:
    image: bde2020/hive:2.3.2-postgresql-metastore
    volumes:
      - testdata:/test-data
      - /tmp/paimon-e2e-tests-jars:/jars
    networks:
      testnetwork:
        aliases:
          - hive-server
    env_file:
      - ./hadoop-hive.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
    ports:
      - "10000"

  hive-metastore:
    image: bde2020/hive:2.3.2-postgresql-metastore
    volumes:
      - testdata:/test-data
    networks:
      testnetwork:
        aliases:
          - hive-metastore
    env_file:
      - ./hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
    ports:
      - "9083"

  hive-metastore-postgresql:
    image: bde2020/hive-metastore-postgresql:2.3.0
    volumes:
      - testdata:/test-data
    networks:
      testnetwork:
        aliases:
          - hive-metastore-postgresql

  # ----------------------------------------
  # Spark services, copied and modified from https://github.com/big-data-europe/docker-spark
  # ----------------------------------------

  spark-master:
    image: bde2020/spark-master:${test.spark.version}-hadoop3.3
    volumes:
      - testdata:/test-data
      - /tmp/paimon-e2e-tests-jars:/jars
    ports:
      - "8080:8080"
      - "7077:7077"
    entrypoint: /bin/bash -c "cp /jars/paimon-spark.jar /spark/jars/ && sh /master.sh"
    environment:
      - INIT_DAEMON_STEP=setup_spark

  spark-worker:
    image: bde2020/spark-worker:${test.spark.version}-hadoop3.3
    depends_on:
      - spark-master
    volumes:
      - testdata:/test-data
      - /tmp/paimon-e2e-tests-jars:/jars
    ports:
      - "8081:8081"
    entrypoint: /bin/bash -c "cp /jars/paimon-spark.jar /spark/jars/ && sh /worker.sh"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"

volumes:
  testdata:
  namenode:
  datanode:

networks:
  testnetwork:
    name: ${NETWORK_ID}
    external: true
