{"id":65402,"date":"2024-06-06T13:53:02","date_gmt":"2024-06-06T10:23:02","guid":{"rendered":"https:\/\/nabfollower.com\/blog\/practical-way-to-use-aws-glue-with-postgresql-1887\/"},"modified":"2024-06-06T13:53:02","modified_gmt":"2024-06-06T10:23:02","slug":"practical-way-to-use-aws-glue-with-postgresql-1887","status":"publish","type":"post","link":"https:\/\/nabfollower.com\/blog\/practical-way-to-use-aws-glue-with-postgresql-1887\/","title":{"rendered":"\u0631\u0648\u0634 \u0639\u0645\u0644\u06cc \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u0627\u0632 \u0686\u0633\u0628 AWS \u0628\u0627 Postgresql"},"content":{"rendered":"
\n

AWS Glue \u06cc\u06a9 \u067e\u0644\u062a \u0641\u0631\u0645 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0628\u062f\u0648\u0646 \u0633\u0631\u0648\u0631 \u0648 \u0631\u0648\u06cc\u062f\u0627\u062f \u0645\u062d\u0648\u0631 \u0627\u0633\u062a \u06a9\u0647 \u062a\u0648\u0633\u0637 \u0622\u0645\u0627\u0632\u0648\u0646 \u0628\u0647 \u0639\u0646\u0648\u0627\u0646 \u0628\u062e\u0634\u06cc \u0627\u0632 \u062e\u062f\u0645\u0627\u062a \u0648\u0628 \u0622\u0645\u0627\u0632\u0648\u0646 \u0627\u0631\u0627\u0626\u0647 \u0634\u062f\u0647 \u0627\u0633\u062a. \u0627\u06cc\u0646 \u06cc\u06a9 \u0633\u0631\u0648\u06cc\u0633 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0627\u0633\u062a \u06a9\u0647 \u06a9\u062f \u0631\u0627 \u062f\u0631 \u067e\u0627\u0633\u062e \u0628\u0647 \u0631\u0648\u06cc\u062f\u0627\u062f\u0647\u0627 \u0627\u062c\u0631\u0627 \u0645\u06cc \u06a9\u0646\u062f \u0648 \u0628\u0647 \u0637\u0648\u0631 \u062e\u0648\u062f\u06a9\u0627\u0631 \u0645\u0646\u0627\u0628\u0639 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632 \u0622\u0646 \u06a9\u062f \u0631\u0627 \u0645\u062f\u06cc\u0631\u06cc\u062a \u0645\u06cc \u06a9\u0646\u062f.<\/p>\n

\u0628\u0647 \u0639\u0646\u0648\u0627\u0646 \u06cc\u06a9 \u0633\u0631\u0648\u06cc\u0633 \u0645\u062d\u0628\u0648\u0628 ETL\u060c Glue \u06af\u0632\u06cc\u0646\u0647 \u0647\u0627\u06cc \u0645\u062a\u0639\u062f\u062f\u06cc \u0631\u0627 \u0628\u0631\u0627\u06cc \u0627\u062a\u0635\u0627\u0644 \u0628\u0647 \u067e\u0627\u06cc\u06af\u0627\u0647 \u0647\u0627\u06cc \u062f\u0627\u062f\u0647 \u0645\u062e\u062a\u0644\u0641\u060c \u0627\u0632 \u062c\u0645\u0644\u0647 PostgreSQL\u060c \u06a9\u0647 \u06cc\u06a9 RDBMS \u067e\u0631\u06a9\u0627\u0631\u0628\u0631\u062f \u0627\u0633\u062a\u060c \u0627\u0631\u0627\u0626\u0647 \u0645\u06cc \u062f\u0647\u062f.<\/p>\n

\u0686\u0633\u0628 \u0686\u0646\u062f\u06cc\u0646 \u0631\u0627\u0647 \u0628\u0631\u0627\u06cc \u062a\u0646\u0638\u06cc\u0645 \u0641\u0631\u0622\u06cc\u0646\u062f\u0647\u0627\u06cc ETL (Extract, Transform, Load) \u0627\u0631\u0627\u0626\u0647 \u0645\u06cc \u062f\u0647\u062f \u06a9\u0647 \u062f\u0631 \u0632\u06cc\u0631 \u0646\u0634\u0627\u0646 \u062f\u0627\u062f\u0647 \u0634\u062f\u0647 \u0627\u0633\u062a:<\/p>\n

<\/p>\n

\u0628\u0627 \u062a\u0646\u0638\u06cc\u0645 \u0628\u0635\u0631\u06cc \u0622\u0646\u060c \u0627\u0646\u062c\u0627\u0645 \u0648\u0638\u0627\u06cc\u0641 ETL \u0628\u0633\u06cc\u0627\u0631 \u0622\u0633\u0627\u0646 \u062a\u0631 \u0645\u06cc \u0634\u0648\u062f.<\/p>\n

\"\u062c\u0632\u0626\u06cc\u0627\u062a<\/p>\n

\u0628\u0631\u0627\u06cc \u0627\u06cc\u062c\u0627\u062f \u06cc\u06a9 \u06a9\u0627\u0631 ETL \u06a9\u0647 \u0628\u0647 \u062a\u0628\u062f\u06cc\u0644 \u062f\u0627\u062f\u0647 \u0647\u0627 \u0627\u0632 \u0648\u0631\u0648\u062f\u06cc S3 \u0628\u0647 \u062e\u0631\u0648\u062c\u06cc PostgreSQL \u06a9\u0645\u06a9 \u0645\u06cc \u06a9\u0646\u062f\u060c \u0641\u0642\u0637 \u0628\u0647 \u0686\u0646\u062f \u06a9\u0644\u06cc\u06a9 \u0646\u06cc\u0627\u0632 \u062f\u0627\u0631\u06cc\u062f.<\/p>\n

\"\u062c\u0632\u0626\u06cc\u0627\u062a<\/p>\n

\u0628\u0627 \u0627\u06cc\u0646 \u062d\u0627\u0644\u060c \u0627\u06cc\u0646 \u0631\u0627\u0647\u200c\u0627\u0646\u062f\u0627\u0632\u06cc \u0645\u062d\u062f\u0648\u062f\u06cc\u062a\u200c\u0647\u0627\u06cc \u0645\u062a\u0639\u062f\u062f\u06cc \u062f\u0627\u0631\u062f\u060c \u0632\u06cc\u0631\u0627 \u0642\u0628\u0644 \u0627\u0632 \u0627\u06cc\u062c\u0627\u062f \u06cc\u06a9 \u06a9\u0627\u0631 ETL \u0628\u0627 \u0639\u0645\u0644\u06a9\u0631\u062f \u0645\u0646\u0627\u0633\u0628\u060c \u0628\u0627\u06cc\u062f \u062a\u0645\u0627\u0645 \u06af\u0632\u06cc\u0646\u0647\u200c\u0647\u0627\u06cc \u0645\u0648\u062c\u0648\u062f \u0631\u0627 \u062f\u0646\u0628\u0627\u0644 \u06a9\u0646\u06cc\u062f.<\/p>\n

\u0627\u06af\u0631 \u0628\u0647 \u062f\u0646\u0628\u0627\u0644 \u0627\u0646\u0639\u0637\u0627\u0641\u200c\u067e\u0630\u06cc\u0631\u06cc \u0628\u06cc\u0634\u062a\u0631 \u062f\u0631 \u067e\u06cc\u06a9\u0631\u0628\u0646\u062f\u06cc \u0647\u0633\u062a\u06cc\u062f\u060c \u0645\u06cc\u200c\u062a\u0648\u0627\u0646\u06cc\u062f \u0627\u0632 \u062a\u0646\u0638\u06cc\u0645\u0627\u062a \u0627\u0633\u06a9\u0631\u06cc\u067e\u062a \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u06a9\u0646\u06cc\u062f.<\/p>\n

\u0628\u0627 \u062a\u0646\u0638\u06cc\u0645 \u0627\u0633\u06a9\u0631\u06cc\u067e\u062a\u060c \u0645\u06cc \u062a\u0648\u0627\u0646\u06cc\u062f \u0645\u0633\u062a\u0642\u06cc\u0645\u0627\u064b \u0627\u0632 \u0627\u0633\u06a9\u0631\u06cc\u067e\u062a \u0628\u0647 \u0645\u0646\u0628\u0639 \u062f\u0627\u062f\u0647 \u06cc\u0627 \u062e\u0631\u0648\u062c\u06cc \u062e\u0648\u062f \u0645\u062a\u0635\u0644 \u0634\u0648\u06cc\u062f. \u0628\u0631\u0627\u06cc \u0627\u0646\u062c\u0627\u0645 \u0627\u06cc\u0646 \u06a9\u0627\u0631\u060c \u0645\u0637\u0627\u0628\u0642 \u0634\u06a9\u0644 \u0632\u06cc\u0631 \u0627\u0632 \u062a\u0646\u0638\u06cc\u0645\u0627\u062a \u062a\u0635\u0648\u06cc\u0631\u06cc \u0628\u0647 \u0635\u0641\u062d\u0647 \u0627\u0633\u06a9\u0631\u06cc\u067e\u062a \u0633\u0648\u0626\u06cc\u0686 \u06a9\u0646\u06cc\u062f:<\/p>\n

\"\u0635\u0641\u062d\u0647<\/p>\n

\u0628\u0631\u0627\u06cc \u06a9\u062f\u060c \u0645\u06cc \u062a\u0648\u0627\u0646\u06cc\u062f \u0627\u0632 \u0627\u0633\u06a9\u0631\u06cc\u067e\u062a \u0647\u0627\u06cc \u0633\u0627\u062f\u0647 \u0645\u0627\u0646\u0646\u062f \u0632\u06cc\u0631 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u06a9\u0646\u06cc\u062f:<\/p>\n

\n
import sys\nfrom awsglue.transforms import *\nfrom awsglue.utils import getResolvedOptions\nfrom pyspark.context import SparkContext\nfrom awsglue.context import GlueContext\nfrom awsglue.job import Job\nimport boto3\n\n# Initialize Glue context and job\nargs = getResolvedOptions(sys.argv, ['JOB_NAME'])\nsc = SparkContext()\nglueContext = GlueContext(sc)\nspark = glueContext.spark_session\njob = Job(glueContext)\njob.init(args['JOB_NAME'], args)\n\n# Read data from S3\ns3_path=\"s3:\/\/your-S3-REPO\/\"\ndatasource = glueContext.create_dynamic_frame.from_options(\n    connection_type=\"s3\",\n    connection_options={\"paths\": [s3_path]},\n    format=\"csv\",  # Adjust format as necessary\n    format_options={\"withHeader\": True, \"separator\": \",\"}\n)\n\ndatasource.printSchema()\n\n# Transform data if needed (this is a simple pass-through in this example)\ntransformed = ApplyMapping.apply(\n    frame = datasource, \n    mappings = [\n        (\"id\", \"string\", \"id\", \"int\"),\n        (\"name\", \"string\", \"name\", \"string\"),\n        (\"age\", \"string\", \"age\", \"int\")\n    ]\n)\n\ntransformed.printSchema()\n\n# Write data to PostgreSQL\nglueContext.write_dynamic_frame.from_options(\n    frame = transformed,\n    connection_type = \"postgresql\",\n    connection_options = {\n        \"url\": \"jdbc:postgresql:\/\/your-PostgresqlDB-Endpoint\",\n        \"dbtable\": \"your_table\",\n        \"user\": \"your-Posgresql-User\",\n        \"password\": \"your-Posgresql-Password\"\n    }\n)\n\n# Commit the job\njob.commit()\n<\/code><\/pre>\n
\n
\n    \u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    \n<\/svg><\/p>\n
    \u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    \n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n
\u0648 \u0628\u0631\u0627\u06cc \u0648\u0631\u0648\u062f\u06cc\u060c \u0645\u06cc \u062a\u0648\u0627\u0646\u06cc\u062f \u0627\u0632 \u06cc\u06a9 \u0641\u0627\u06cc\u0644 \u0628\u0627 \u0641\u0631\u0645\u062a CSV \u0645\u0627\u0646\u0646\u062f \u0627\u06cc\u0646 \u0627\u0633\u062a\u0641\u0627\u062f\u0647 \u06a9\u0646\u06cc\u062f:<\/p>\n
\n
id,name,age\n1,John Doe,30\n2,Jane Smith, 15\n3,Bob Yellow,20\n4,Roshan Brown,18\n5,Bam Black,55\n<\/code><\/pre>\n
\n
\n    \u0648\u0627\u0631\u062f \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u0634\u0648\u06cc\u062f<\/title>\n    \n<\/svg><\/p>\n
    \u0627\u0632 \u062d\u0627\u0644\u062a \u062a\u0645\u0627\u0645 \u0635\u0641\u062d\u0647 \u062e\u0627\u0631\u062c \u0634\u0648\u06cc\u062f<\/title>\n    \n<\/svg><\/p>\n<\/div>\n<\/div>\n<\/div>\n
\u067e\u0633 \u0627\u0632 \u0622\u0646\u060c \u0645\u06cc \u062a\u0648\u0627\u0646\u06cc\u062f \u06a9\u0627\u0631 \u0631\u0627 \u0634\u0631\u0648\u0639 \u06a9\u0646\u06cc\u062f \u0648 \u0645\u0646\u062a\u0638\u0631 \u0628\u0645\u0627\u0646\u06cc\u062f \u062a\u0627 \u062a\u0645\u0627\u0645 \u0634\u0648\u062f.  \u062f\u0631 \u0635\u0648\u0631\u062a \u0645\u0648\u0641\u0642\u06cc\u062a\u060c \u0645\u0637\u0627\u0628\u0642 \u0634\u06a9\u0644 \u0632\u06cc\u0631:<\/p>\n
\"\u0648\u0636\u0639\u06cc\u062a<\/p>\n
\u0645\u06cc \u062a\u0648\u0627\u0646\u06cc\u062f \u0622\u062e\u0631\u06cc\u0646 \u0646\u062a\u06cc\u062c\u0647 \u0631\u0627 \u062f\u0631 posgresql \u062e\u0648\u062f \u0628\u0631\u0631\u0633\u06cc \u06a9\u0646\u06cc\u062f.<\/p>\n
\u0641\u06a9\u0631 \u0645\u06cc \u06a9\u0646\u0645 \u0641\u0639\u0644\u0627\u064b \u0628\u0631\u0627\u06cc \u0627\u06cc\u0646 \u0645\u0642\u0627\u0644\u0647 \u0647\u0645\u06cc\u0646 \u0627\u0633\u062a.  \u062f\u0631 \u0645\u0648\u0631\u062f \u0627\u0641\u06a9\u0627\u0631 \u062e\u0648\u062f \u062f\u0631 \u0632\u06cc\u0631 \u0646\u0638\u0631 \u062f\u0647\u06cc\u062f!  \u0628\u0627 \u062a\u0634\u06a9\u0631.<\/p>\n<\/p><\/div>\n","protected":false},"excerpt":{"rendered":"
AWS Glue \u06cc\u06a9 \u067e\u0644\u062a \u0641\u0631\u0645 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0628\u062f\u0648\u0646 \u0633\u0631\u0648\u0631 \u0648 \u0631\u0648\u06cc\u062f\u0627\u062f \u0645\u062d\u0648\u0631 \u0627\u0633\u062a \u06a9\u0647 \u062a\u0648\u0633\u0637 \u0622\u0645\u0627\u0632\u0648\u0646 \u0628\u0647 \u0639\u0646\u0648\u0627\u0646 \u0628\u062e\u0634\u06cc \u0627\u0632 \u062e\u062f\u0645\u0627\u062a \u0648\u0628 \u0622\u0645\u0627\u0632\u0648\u0646 \u0627\u0631\u0627\u0626\u0647 \u0634\u062f\u0647 \u0627\u0633\u062a. \u0627\u06cc\u0646 \u06cc\u06a9 \u0633\u0631\u0648\u06cc\u0633 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0627\u0633\u062a \u06a9\u0647 \u06a9\u062f \u0631\u0627 \u062f\u0631 \u067e\u0627\u0633\u062e \u0628\u0647 \u0631\u0648\u06cc\u062f\u0627\u062f\u0647\u0627 \u0627\u062c\u0631\u0627 \u0645\u06cc \u06a9\u0646\u062f \u0648 \u0628\u0647 \u0637\u0648\u0631 \u062e\u0648\u062f\u06a9\u0627\u0631 \u0645\u0646\u0627\u0628\u0639 \u0645\u062d\u0627\u0633\u0628\u0627\u062a\u06cc \u0645\u0648\u0631\u062f \u0646\u06cc\u0627\u0632 \u0622\u0646 \u06a9\u062f \u0631\u0627 \u0645\u062f\u06cc\u0631\u06cc\u062a \u0645\u06cc \u06a9\u0646\u062f. \u0628\u0647 …<\/p>\n","protected":false},"author":2,"featured_media":65403,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"fifu_image_url":"","fifu_image_alt":"","footnotes":""},"categories":[339],"tags":[],"class_list":["post-65402","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-dev"],"_links":{"self":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts\/65402","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/comments?post=65402"}],"version-history":[{"count":0,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/posts\/65402\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/media\/65403"}],"wp:attachment":[{"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/media?parent=65402"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/categories?post=65402"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/nabfollower.com\/blog\/wp-json\/wp\/v2\/tags?post=65402"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}