# Welcome to your new notebook
# Type here in the cell editor to add code!
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pyspark.sql.functions as F
import pyspark.sql.types as T

df = spark.sql("SELECT * FROM lakehouse_1_lab.sales LIMIT 1000")

df.columns

display(df.describe())

display(df.summary())

# Convert the 'OrderDate' column to a date type
df = df.withColumn('OrderDate', F.to_date('OrderDate').cast(T.DateType()))

# Group the data by 'OrderDate' and calculate the sum of 'TaxAmount' and 'UnitPrice'
grouped_df = df.groupBy('OrderDate').agg(F.sum('TaxAmount').alias('TaxAmount'), F.sum('UnitPrice').alias('Price'))

# Convert the DataFrame to Pandas DataFrame for plotting
pandas_df = grouped_df.toPandas()

# Plotting the data
plt.plot(pandas_df['OrderDate'], pandas_df['TaxAmount'], label='Tax')
plt.plot(pandas_df['OrderDate'], pandas_df['Price'], label='Price')
plt.legend()
plt.title('Price and Tax Amount Relationship to Order Date')
plt.ylabel('TaxAmount and Price')
plt.xlabel('Order Date')
plt.show()

pandas_df.plot(x='OrderDate', y='TaxAmount', label='Tax', kind='line')
plt.legend()
plt.title('Tax Amount Relationship to Order Date')
plt.ylabel('TaxAmount')
plt.xlabel('Order Date')
plt.show()

pandas_df.plot(x='OrderDate', y='Price', label='Price', kind='line')
plt.legend()
plt.title('Price Relationship to Order Date')
plt.ylabel('Price')
plt.xlabel('Order Date')
plt.show()