بررسی اسکریپت‌های راه‌اندازی شل در گردش‌های کاری مدیریت‌شده برای Apache Airflow (MWAA) و mwaa-local-runner

from datetime import datetime
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator
import os


def print_hello():
 print(os.environ['BUILD_ON_OPEN_SOURCE'])
 return 'os.environ["BUILD_ON_OPEN_SOURCE"]'

dag = DAG('startup_script_demo', description='Showing Startup Scripts', schedule_interval=None, start_date=datetime(2017, 3, 20), catchup=False)

java_env = BashOperator(
 task_id="java_env",
 bash_command="java --version",
 dag=dag)

env_variable = PythonOperator(
 task_id='env_variable',
 python_callable=print_hello,
 dag=dag)

java_env >> env_variable

  File "/usr/local/airflow/dags/simple-dag.py", line 9, in print_hello
    print(os.environ['BUILD_ON_OPEN_SOURCE'])
  File "/usr/lib/python3.10/os.py", line 680, in __getitem__
    raise KeyError(key) from None
KeyError: 'BUILD_ON_OPEN_SOURCE'
[2023-04-14, 13:13:04 UTC] {{taskinstance.py:1401}} INFO - Marking task as FAILED. 

[2023-04-14, 13:12:29 UTC] {{subprocess.py:75}} INFO - Running command: ['/usr/bin/bash', '-c', 'java --version']
[2023-04-14, 13:12:29 UTC] {{subprocess.py:86}} INFO - Output:
[2023-04-14, 13:12:29 UTC] {{subprocess.py:93}} INFO - Unrecognized option: --version
[2023-04-14, 13:12:29 UTC] {{subprocess.py:93}} INFO - Error: Could not create the Java Virtual Machine.
[2023-04-14, 13:12:29 UTC] {{subprocess.py:93}} INFO - Error: A fatal exception has occurred. Program will exit.
[2023-04-14, 13:12:29 UTC] {{subprocess.py:97}} INFO - Command exited with return code 1
[2023-04-14, 13:12:29 UTC] {{taskinstance.py:1851}} ERROR - Task failed with exception
Traceback (most recent call last):
  File "/usr/local/airflow/.local/lib/python3.10/site-packages/airflow/operators/bash.py", line 196, in execute
    raise AirflowException(
airflow.exceptions.AirflowException: Bash command failed. The command returned a non-zero exit code 1.

    install_requirements
    execute_startup_script
    source stored_env    <-- added this
    airflow db init

./mwaa-local-env build-image

#!/bin/sh

echo "Running sample startup script."

export BUILD_ON_OPEN_SOURCE="awesome"

sudo yum install java-17-amazon-corretto-headless -y

./mwaa-local-env test-startup-script

Container amazon/mwaa-local:2_4 exists. Skipping build
Running sample startup script.
awesome
Loaded plugins: ovl, priorities
amzn2-core                                                                                                                                                                       | 3.7 kB  00:00:00     
(1/3): amzn2-core/2/x86_64/group_gz                                                                                                                                              | 2.5 kB  00:00:00     
(2/3): amzn2-core/2/x86_64/updateinfo                                                                                                                                            | 586 kB  00:00:00     
(3/3): amzn2-core/2/x86_64/primary_db                                                                                                                                            |  71 MB  00:00:24     
Resolving Dependencies
--> Running transaction check
---> Package java-17-amazon-corretto-headless.x86_64 1:17.0.6+10-1.amzn2.1 will be installed
--> Processing Dependency: dejavu-sans-mono-fonts for package: 1:java-17-amazon-corretto-headless-17.0.6+10-1.amzn2.1.x86_64
--> Processing Dependency: dejavu-serif-fonts for package: 1:java-17-amazon-corretto-headless-17.0.6+10-1.amzn2.1.x86_64
--> Running transaction check
---> Package dejavu-sans-mono-fonts.noarch 0:2.33-6.amzn2 will be installed
---> Package dejavu-serif-fonts.noarch 0:2.33-6.amzn2 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

===============================================================================================================================================================================
 Package                                                        Arch                                 Version                                             Repository                                Size
===============================================================================================================================================================================
Installing:
 java-17-amazon-corretto-headless                               x86_64                               1:17.0.6+10-1.amzn2.1                               amzn2-core                                94 M
Installing for dependencies:
 dejavu-sans-mono-fonts                                         noarch                               2.33-6.amzn2                                        amzn2-core                               433 k
 dejavu-serif-fonts                                             noarch                               2.33-6.amzn2                                        amzn2-core                               777 k

Transaction Summary
===============================================================================================================================================================================
Install  1 Package (+2 Dependent packages)

Total download size: 95 M
Installed size: 240 M
Is this ok [y/d/N]: ^C/entrypoint.sh: line 70: stored_env: No such file or directory
(base)  @094459  ~/Projects/airflow/cicd/demos/local-airflow-dev/mwaa-local  ./mwaa-local-env test-startup-script
Container amazon/mwaa-local:2_4 exists. Skipping build
Running sample startup script.
Loaded plugins: ovl, priorities
amzn2-core                                                                                                                                                                       | 3.7 kB  00:00:00     
(1/3): amzn2-core/2/x86_64/group_gz                                                                                                                                              | 2.5 kB  00:00:00     
(2/3): amzn2-core/2/x86_64/updateinfo                                                                                                                                            | 586 kB  00:00:00     
(3/3): amzn2-core/2/x86_64/primary_db                                                                                                                                            |  71 MB  00:00:22     
Resolving Dependencies
--> Running transaction check
---> Package java-17-amazon-corretto-headless.x86_64 1:17.0.6+10-1.amzn2.1 will be installed
--> Processing Dependency: dejavu-sans-mono-fonts for package: 1:java-17-amazon-corretto-headless-17.0.6+10-1.amzn2.1.x86_64
--> Processing Dependency: dejavu-serif-fonts for package: 1:java-17-amazon-corretto-headless-17.0.6+10-1.amzn2.1.x86_64
--> Running transaction check
---> Package dejavu-sans-mono-fonts.noarch 0:2.33-6.amzn2 will be installed
---> Package dejavu-serif-fonts.noarch 0:2.33-6.amzn2 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

Installing:
 java-17-amazon-corretto-headless                               x86_64                               1:17.0.6+10-1.amzn2.1                               amzn2-core                                94 M
Installing for dependencies:
 dejavu-sans-mono-fonts                                         noarch                               2.33-6.amzn2                                        amzn2-core                               433 k
 dejavu-serif-fonts                                             noarch                               2.33-6.amzn2                                        amzn2-core                               777 k

Transaction Summary
===============================================================================================================================================================================
Install  1 Package (+2 Dependent packages)

Total download size: 95 M
Installed size: 240 M
Downloading packages:
(1/3): dejavu-sans-mono-fonts-2.33-6.amzn2.noarch.rpm                                                                                                                            | 433 kB  00:00:00     
(2/3): dejavu-serif-fonts-2.33-6.amzn2.noarch.rpm                                                                                                                                | 777 kB  00:00:00     
(3/3): java-17-amazon-corretto-headless-17.0.6+10-1.amzn2.1.x86_64.rpm                                                                                                           |  94 MB  00:00:30     


Complete!
Finished running startup script. Execution time: 75.612s.
Running verification
Verification completed

No package java-17-amazon-corretto-headlesss available.
Error: Nothing to do
Finished running startup script. Execution time: 29.105s.
Running verification
Verification completed

./mwaa-local-env start
..
..
aws-mwaa-local-runner-2_4-local-runner-1  | Requirement already satisfied: anyio==3.* in ./.local/lib/python3.10/site-packages (from httpcore<0.16.0,>=0.15.0->httpx->apache-airflow>=2.3.0->apache-airflow-providers-amazon==7.3.0->-r /usr/local/airflow/requirements/requirements.txt (line 4)) (3.6.2)
aws-mwaa-local-runner-2_4-local-runner-1  | Requirement already satisfied: h11<0.13,>=0.11 in ./.local/lib/python3.10/site-packages (from httpcore<0.16.0,>=0.15.0->httpx->apache-airflow>=2.3.0->apache-airflow-providers-amazon==7.3.0->-r /usr/local/airflow/requirements/requirements.txt (line 4)) (0.12.0)
aws-mwaa-local-runner-2_4-local-runner-1  | Running sample startup script.
aws-mwaa-local-runner-2_4-local-runner-1  | Loaded plugins: ovl, priorities
aws-mwaa-local-runner-2_4-local-runner-1  | Resolving Dependencies
aws-mwaa-local-runner-2_4-local-runner-1  | --> Running transaction check
aws-mwaa-local-runner-2_4-local-runner-1  | ---> Package java-17-amazon-corretto-headless.x86_64 1:17.0.6+10-1.amzn2.1 will be installed
..

[2023-04-17, 18:13:29 UTC]  {{logging_mixin.py:137}} اطلاعات - عالی
[2023-04-17, 18:13:29 UTC] {{python.py:177}} اطلاعات - انجام شد.  مقدار برگشتی: os.environ بود["BUILD_ON_OPEN_SOURCE"]
[2023-04-17, 18:13:29 UTC]  {{taskinstance.py:1401}} اطلاعات - علامت گذاری کار به عنوان SUCCESS.  dag_id=startup_script_demo، task_id=env_variable، execution_date=20230417T181324، start_date=20230417T181328، end_date=20230417T181329
[2023-04-17, 18:13:29 UTC] {{local_task_job.py:159}} اطلاعات - کار با کد برگشتی 0 خارج شد
[

[2023-04-14, 13:56:05 UTC] {{subprocess.py:75}} اطلاعات - دستور در حال اجرا: ['/usr/bin/bash', '-c', 'java --version']
[2023-04-14, 13:56:05 UTC]  {{subprocess.py:86}} اطلاعات - خروجی:
[2023-04-14, 13:56:05 UTC] {{subprocess.py:93}} اطلاعات - openjdk 17.0.6 2023-01-17 LTS
[2023-04-14, 13:56:05 UTC] {{subprocess.py:93}} اطلاعات - OpenJDK Runtime Environment Corretto-17.0.6.10.1 (build 17.0.6+10-LTS)
[2023-04-14, 13:56:05 UTC] {{subprocess.py:93}} اطلاعات - OpenJDK 64-bit Server VM Corretto-17.0.6.10.1 (build 17.0.6+10-LTS، حالت ترکیبی، اشتراک گذاری)
[2023-04-14, 13:56:05 UTC] {{subprocess.py:97}} اطلاعات - فرمان با کد برگشتی 0 خارج شد