AWS GLUE에서 다른 GLUE 실행하기

AWS GLUE에서 다른 GLUE 실행하기

 

import boto3
import time
from botocore.exceptions import ClientError

def wait_for_job_completion(glue_client, job_name, run_id):
    """
    Wait for an AWS Glue job to complete and return the status.
    
    Args:
        glue_client: Boto3 Glue client
        job_name: Name of the Glue job
        run_id: ID of the specific job run
        
    Returns:
        str: Final status of the job rund
    """
    while True:
        try:
            response = glue_client.get_job_run(
                JobName=job_name,
                RunId=run_id
            )
            status = response['JobRun']['JobRunState']
            
            if status in ['SUCCEEDED', 'FAILED', 'TIMEOUT', 'ERROR', 'STOPPED']:
                print(f"Job {job_name} finished with status: {status}")
                if status != 'SUCCEEDED':
                    error_message = response['JobRun'].get('ErrorMessage', 'No error message available')
                    print(f"Error details: {error_message}")
                return status
                
            print(f"Job {job_name} is still running with status: {status}")
            time.sleep(30)  # Wait for 30 seconds before checking again
            
        except ClientError as e:
            print(f"Error getting job run status: {e}")
            raise

def run_glue_job(job_name, arguments):
    """
    Run an AWS Glue job and wait for completion.
    
    Args:
        job_name: Name of the Glue job
        arguments: Dictionary of job arguments
        
    Returns:
        tuple: (bool indicating success, job run ID)
    """
    glue_client = boto3.client('glue')
    
    try:
        # Start the Glue job
        response = glue_client.start_job_run(
            JobName=job_name,
            Arguments=arguments
        )
        job_run_id = response['JobRunId']
        print(f"Started Glue job {job_name} with run ID: {job_run_id}")
        
        # Wait for job completion
        final_status = wait_for_job_completion(glue_client, job_name, job_run_id)
        
        return final_status == 'SUCCEEDED', job_run_id
        
    except ClientError as e:
        print(f"Error starting/monitoring Glue job: {e}")
        raise





# 글루잡 매뉴얼 실행
jobName='etl-glue-job-resource'
jobArguments = {
    '--param_01': '2025-01-01',
    '--param_02': 'TEST.TXT'
}
print(f"Running Glue job {jobName} with arguments: {jobArguments}")


try:
    success, run_id = run_glue_job(jobName, jobArguments)
    
    if success:
        print(f"Job completed successfully! Run ID: {run_id}")
    else:
        print(f"Job failed! Run ID: {run_id}")
        
except Exception as e:
    print(f"An error occurred: {e}")