Environment Setup

docker --version
docker-compose --version
docker run hello-world

java -version
javac -version
echo $JAVA_HOME

version: '3.8'

# Set FLINK_IMAGE_TAG in your environment or .env file:
#   Flink 1.20: FLINK_IMAGE_TAG=1.20-java11
#   Flink 2.2:  FLINK_IMAGE_TAG=2.2-java17

services:
  # Flink JobManager
  jobmanager:
    image: flink:${FLINK_IMAGE_TAG:-2.2-java17}
    hostname: jobmanager
    container_name: flink-jobmanager
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=

# Create Managed Service for Apache Flink project directory
mkdir my-msf-app
cd my-msf-app

# Create required directories
mkdir -p flink-jobs src/main/java src/test/java

# Copy docker-compose.yml to project root

# Start all services
docker-compose up -d

# Verify services are running
docker-compose ps

# Check taskmanager logs for submitted Flink jobs 
docker-compose logs taskmanager

# Check Flink Web UI
open http://localhost:8081

# Check Kafka
docker-compose exec kafka kafka-topics --bootstrap-server localhost:9092 --list

# For Flink 1.20 (Java 11):
mvn clean package -Plocal -q

# For Flink 2.2 (Java 17):
# Set JAVA_HOME to your Java 17 installation path before building
export JAVA_HOME=<path-to-java-17>
mvn clean package -Plocal -q

cp target/my-flink-app-1.0.jar flink-jobs/

docker exec flink-jobmanager flink run /opt/flink/jobs/my-flink-app-1.0.jar

# List Managed Service for Apache Flink applications
aws kinesisanalyticsv2 list-applications

# Describe Managed Service for Apache Flink application
aws kinesisanalyticsv2 describe-application --application-name my-app

# Create savepoint
aws kinesisanalyticsv2 create-application-snapshot --application-name my-app --snapshot-name my-snapshot