Docker コンテナを使って AWS Glue 5.0 のジョブをローカルで開発・テストする

PROFILE_NAME="profile_name"

docker pull public.ecr.aws/glue/aws-glue-libs:5

$ WORKSPACE_LOCATION=/local_path_to_workspace 
$ SCRIPT_FILE_NAME=sample.py 
$ mkdir -p ${WORKSPACE_LOCATION}/src 
$ vim ${WORKSPACE_LOCATION}/src/${SCRIPT_FILE_NAME}

$ docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_spark_submit \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    spark-submit /home/hadoop/workspace/src/$SCRIPT_FILE_NAME

$ docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_pyspark \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    pyspark

Python 3.11.6 (main, Jan  9 2025, 00:00:00) [GCC 11.4.1 20230605 (Red Hat 11.4.1-2)] on linux 
 Type "help", "copyright", "credits" or "license" for more information.
 Setting default log level to "WARN".
 To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
 Welcome to 
      ____              __ 
     / __/__  ___ _____/ /__ 
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 3.5.2-amzn-1 
      /_/

 Using Python version 3.11.6 (main, Jan  9 2025 00:00:00)
 Spark context Web UI available at None 
 Spark context available as 'sc' (master = local[*], app id = local-1740643079929).
 SparkSession available as 'spark'.
>>> 

$ WORKSPACE_LOCATION=/local_path_to_workspace 
$ SCRIPT_FILE_NAME=sample.py 
$ UNIT_TEST_FILE_NAME=test_sample.py 
$ mkdir -p ${WORKSPACE_LOCATION}/tests 
$ vim ${WORKSPACE_LOCATION}/tests/${UNIT_TEST_FILE_NAME}

$ docker run -i --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    --workdir /home/hadoop/workspace \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_pytest \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    -c "python3 -m pytest --disable-warnings"

============================= test session starts ==============================
 platform linux -- Python 3.11.6, pytest-8.3.4, pluggy-1.5.0 
 rootdir: /home/hadoop/workspace 
 plugins: integration-mark-0.2.0 
 collected 1 item 

 tests/test_sample.py .                                                   [100%] 

======================== 1 passed, 1 warning in 34.28s =========================

{
    "python.defaultInterpreterPath": "/usr/bin/python3.11",
    "python.analysis.extraPaths": [ 
        "/usr/lib/spark/python/lib/py4j-0.10.9.7-src.zip:/usr/lib/spark/python/:/usr/lib/spark/python/lib/",
    ] 
}

$ docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_pyspark \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    pyspark

import sys 
 from pyspark.context import SparkContext 
 from awsglue.context import GlueContext 
 from awsglue.job import Job 
 from awsglue.utils import getResolvedOptions 


 class GluePythonSampleTest:
    def __init__(self):
        params = [] 
        if '--JOB_NAME' in sys.argv:
            params.append('JOB_NAME')
        args = getResolvedOptions(sys.argv, params)

        self.context = GlueContext(SparkContext.getOrCreate())
        self.job = Job(self.context)

        if 'JOB_NAME' in args:
            jobname = args['JOB_NAME'] 
        else:
            jobname = "test"
        self.job.init(jobname, args)

    def run(self):
        dyf = read_json(self.context, "s3://awsglue-datasets/examples/us-legislators/all/persons.json")
        dyf.printSchema()

        self.job.commit()


 def read_json(glue_context, path):
    dynamicframe = glue_context.create_dynamic_frame.from_options(
        connection_type='s3',
        connection_options={
            'paths': [path],
            'recurse': True 
        },
        format='json'
    )
    return dynamicframe 


 if __name__ == '__main__':
    GluePythonSampleTest().run()

 import pytest 
 from pyspark.context import SparkContext 
 from awsglue.context import GlueContext 
 from awsglue.job import Job 
 from awsglue.utils import getResolvedOptions 
 import sys 
 from src import sample 


@pytest.fixture(scope="module", autouse=True)
 def glue_context():
    sys.argv.append('--JOB_NAME')
    sys.argv.append('test_count')

    args = getResolvedOptions(sys.argv, ['JOB_NAME'])
    context = GlueContext(SparkContext.getOrCreate())
    job = Job(context)
    job.init(args['JOB_NAME'], args)

$ docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    -v $WORKSPACE_LOCATION/jars/:/opt/spark/jars/ \ 
    --workdir /home/hadoop/workspace \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_jdbc \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    pyspark

$ mkdir -p $WORKSPACE_LOCATION/jupyterlab/
$ cd $WORKSPACE_LOCATION/jupyterlab/
$ vim Dockerfile.livy_jupyter

FROM public.ecr.aws/glue/aws-glue-libs:5 AS glue-base 

 ENV LIVY_SERVER_JAVA_OPTS="--add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"

# Download Livy 
 ADD --chown=hadoop:hadoop https://dlcdn.apache.org/incubator/livy/0.8.0-incubating/apache-livy-0.8.0-incubating_2.12-bin.zip ./

# Install and configure Livy 
 RUN unzip apache-livy-0.8.0-incubating_2.12-bin.zip && \ 
 rm apache-livy-0.8.0-incubating_2.12-bin.zip && \ 
 mv apache-livy-0.8.0-incubating_2.12-bin livy && \ 
 mkdir -p livy/logs && \ 
 cat <> livy/conf/livy.conf 
 livy.server.host = 0.0.0.0 
 livy.server.port = 8998 
 livy.spark.master = local 
 livy.repl.enable-hive-context = true 
 livy.spark.scala-version = 2.12 
 EOF && \ 
 cat <> livy/conf/log4j.properties 
 log4j.rootCategory=INFO,console 
 log4j.appender.console=org.apache.log4j.ConsoleAppender 
 log4j.appender.console.target=System.err 
 log4j.appender.console.layout=org.apache.log4j.PatternLayout 
 log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 
 log4j.logger.org.eclipse.jetty=WARN 
 EOF 

# Switching to root user temporarily to install dev dependency packages 
 USER root 
 RUN dnf update -y && dnf install -y krb5-devel gcc python3.11-devel 
 USER hadoop 

# Install SparkMagic and JupyterLab 
 RUN export PATH=$HOME/.local/bin:$HOME/livy/bin/:$PATH && \ 
 printf "numpy<2\nIPython<=7.14.0\n" > /tmp/constraint.txt && \ 
 pip3.11 --no-cache-dir install --constraint /tmp/constraint.txt --user pytest boto==2.49.0 jupyterlab==3.6.8 IPython==7.14.0 ipykernel==5.5.6 ipywidgets==7.7.2 sparkmagic==0.21.0 jupyterlab_widgets==1.1.11 && \ 
 jupyter-kernelspec install --user $(pip3.11 --no-cache-dir show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel && \ 
 jupyter-kernelspec install --user $(pip3.11 --no-cache-dir show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel && \ 
 jupyter server extension enable --user --py sparkmagic && \ 
 cat <> /home/hadoop/.local/bin/entrypoint.sh 
#!/usr/bin/env bash 
 mkdir -p /home/hadoop/workspace/
 livy-server start 
 sleep 5 
 jupyter lab --no-browser --ip=0.0.0.0 --allow-root --ServerApp.root_dir=/home/hadoop/workspace/ --ServerApp.token='' --ServerApp.password=''
 EOF 

# Setup Entrypoint script 
 RUN chmod + x /home/hadoop/.local/bin/entrypoint.sh 

# Add default SparkMagic Config 
 ADD --chown=hadoop:hadoop https://raw.githubusercontent.com/jupyter-incubator/sparkmagic/refs/heads/master/sparkmagic/example_config.json .sparkmagic/config.json 

# Update PATH var 
 ENV PATH=/home/hadoop/.local/bin:/home/hadoop/livy/bin/:$PATH 

 ENTRYPOINT ["/home/hadoop/.local/bin/entrypoint.sh"] 

docker build \ 
    -t glue_v5_livy \ 
    --file $WORKSPACE_LOCATION/jupyterlab/Dockerfile.livy_jupyter \ 
    $WORKSPACE_LOCATION/jupyterlab/

docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    -p 8998:8998 \ 
    -p 8888:8888 \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_jupyter  \ 
    glue_v5_livy

$ docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    -v $EXTRA_PYTHON_PACKAGE_LOCATION:/home/hadoop/workspace/extra_python_path/ \ 
    --workdir /home/hadoop/workspace \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    --name glue5_pylib \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    -c 'export PYTHONPATH=/home/hadoop/workspace/extra_python_path/:$PYTHONPATH ; pyspark'

Python 3.11.6 (main, Jan  9 2025, 00:00:00) [GCC 11.4.1 20230605 (Red Hat 11.4.1-2)] on linux 
 Type "help", "copyright", "credits" or "license" for more information.
 Setting default log level to "WARN".
 To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
 Welcome to 
      ____              __ 
     / __/__  ___ _____/ /__ 
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 3.5.2-amzn-1 
      /_/

 Using Python version 3.11.6 (main, Jan  9 2025 00:00:00)
 Spark context Web UI available at None 
 Spark context available as 'sc' (master = local[*], app id = local-1740719582296).
 SparkSession available as 'spark'.
>>> import sys 
>>> "/home/hadoop/workspace/extra_python_path" in sys.path 
 True 

docker run -it --rm \ 
    -v ~/.aws:/home/hadoop/.aws \ 
    -v $WORKSPACE_LOCATION:/home/hadoop/workspace/ \ 
    --workdir /home/hadoop/workspace \ 
    -e AWS_PROFILE=$PROFILE_NAME \ 
    -e SCRIPT_FILE_NAME=$SCRIPT_FILE_NAME \ 
    --name glue5_pylib \ 
    public.ecr.aws/glue/aws-glue-libs:5 \ 
    -c 'pip3 install snowflake==1.0.5 ; spark-submit /home/hadoop/workspace/src/$SCRIPT_FILE_NAME'