Glue ETL Migration

{
    "Name": "migrate-to-s3tables",
    "Role": "arn:aws:iam::<account-id>:role/<glue-role>",
    "Command": {
        "Name": "glueetl",
        "ScriptLocation": "s3://<scripts-bucket>/scripts/migrate.py",
        "PythonVersion": "3"
    },
    "DefaultArguments": {
        "--datalake-formats": "iceberg",
        "--enable-glue-datacatalog": "true",
        "--conf": "<see iceberg-catalog-config-and-usage.md for S3 Tables Analytics Integration or REST config>"
    },
    "GlueVersion": "5.1",
    "NumberOfWorkers": 10,
    "WorkerType": "G.1X"
}

aws glue create-job --cli-input-json file://job-config.json --region <region>

import sys
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

args = getResolvedOptions(sys.argv, [
    'JOB_NAME', 'source_database', 'source_table',
    'target_namespace', 'target_table'
])

sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(args['JOB_NAME'], args)

# Read from source (Glue Data Catalog)
source_df = spark.read.table(
    f"glue_catalog.{args['source_database']}.{args['source_table']}"
)

# Apply transforms (customize as needed)
# Example: lowercase column names for S3 Tables compatibility
for col_name in source_df.columns:
    if col_name != col_name.lower():
        source_df = source_df.withColumnRenamed(col_name, col_name.lower())

# Write to S3 Table
target_table = f"s3tablescatalog.{args['target_namespace']}.{args['target_table']}"

source_df.writeTo(target_table) \
    .tableProperty("format-version", "2") \
    .createOrReplace()

# Verify row count
source_count = spark.read.table(
    f"glue_catalog.{args['source_database']}.{args['source_table']}"
).count()
target_count = spark.read.table(target_table).count()
print(f"Source rows: {source_count}, Target rows: {target_count}")

job.commit()

# Start the job
JOB_RUN_ID=$(aws glue start-job-run \
  --job-name "migrate-to-s3tables" \
  --arguments '{"--source_database":"legacy_db","--source_table":"orders","--target_namespace":"analytics","--target_table":"orders"}' \
  --query 'JobRunId' --output text)

# Check status
aws glue get-job-run --job-name "migrate-to-s3tables" --run-id "$JOB_RUN_ID"