PySpark Local Python Windows Notebook را اجرا کنید

import os
os.environ["JAVA_HOME"] = fr"D:\Soft\JAVA\jdk-23.0.1"
os.environ["PATH"] = os.environ["JAVA_HOME"] + "/bin;" + os.environ["PATH"]

import os
os.environ["SPARK_HOME"] = fr"D:\Soft\pyspark\spark-3.5.4-bin-hadoop3"
os.environ["PATH"] = os.environ["SPARK_HOME"] + "/bin;" + os.environ["PATH"]

import numpy as np
import pandas as pd
spark = SparkSession.builder \
    .appName("Debugging Example") \
    .master("local[*]") \
    .config("spark.eventLog.enabled", "true") \
    .config("spark.sql.shuffle.partitions", "1") \
    .getOrCreate()

spark.sparkContext.setLogLevel("DEBUG")
# Enable Arrow-based columnar data transfers
spark.conf.set("spark.sql.execution.arrow.enabled", "true")

# Generate a pandas DataFrame
pdf = pd.DataFrame(np.random.rand(100, 3))

# Create a Spark DataFrame from a pandas DataFrame using Arrow
df = spark.createDataFrame(pdf)
# rename columns
df = df.toDF("a", "b", "c")
df

import requests

# URL for the dataset
url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"

# Download the dataset and save it locally
response = requests.get(url)
with open("iris.data", "wb") as file:
    file.write(response.content)
from pyspark.sql import SparkSession

# Create a SparkSession
spark = SparkSession.builder \
    .appName("Iris Data Analysis") \
    .master("local[*]")\
    .getOrCreate()

# Path to the locally downloaded Iris dataset
iris_data_path = "iris.data"

# Define the schema for the data
columns = ["sepal_length", "sepal_width", "petal_length", "petal_width", "species"]

# Load the data into a DataFrame
df = spark.read.csv(iris_data_path, header=False, inferSchema=True)

# Set column names
df = df.toDF(*columns)

# Show the first few rows of the DataFrame
df.show()

# Stop the SparkSession when done
spark.stop()