jekwatt / idiomatic_pandas Goto Github PK

d1 = {c: normalize_name(c) for c in df.columns}
df.rename(columns=d1, inplace=True)

The proper way to modify df is to apply one of the accessors
.loc[], 
.iloc[], 
.at[], 
or .iat[]

mask = df["A"] > 5
df.loc[mask, 'B'] = 4

Method 1: isin function/method

result1 = df1[~df1.apply(tuple, 1).isin(df2.apply(tuple, 1))]
print(result1)

Method 2: merge

# indicator parameter will insert a new field ("_merge")
result2 = df1.merge(df3, indicator=True, how="outer").loc[lambda v: v["_merge"] != "both"]
print(result2)

result3 = df1.merge(df3, indicator=True, how="outer")
result3[test_merge._merge != "both"]

e.g.
df["col_1"].str.lower()
df["col_1"].str.upper()
df["col_1"] = df["col_1"].str.title()
df["col_1"].str.len()

import glob
import os

from pathlib import Path

today = datetime.now().date()
y, m, d = today.year, today.month, today.day
md = f"{m:02d}-{d:02d}"

p = Path.cwd().parents[0]
here = p / 'data' / 'prod' / md

all_csv_files = sorted(glob(os.path.join(here, "*.csv")))
df_from_each_csv = (pd.read_csv(f) for f in all_csv_files)
df = pd.concat(df_from_each_csv, ignore_index=True)

from pathlib import Path

src_file = Path.cwd() / 'data.xlsx'
df = pd.read_excel(src_file, header=1, usecols='B:F')

# 1
df1 = pd.read_csv('input1.csv')
df1['col_1'] = df1['col_1'].str.strip()

#2
modify your read_csv lines to also use [skipinitialspace=True]
df1 = pd.read_csv('input1.csv', skipinitialspace=True)

http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html

# 3
pd.read_csv(..., converters={'col_1': str.strip})
# only strip leading whitespace
pd.read_csv(..., converters={'col_1': str.lstrip})

grouped_df = df.groupby("col_1")
grouped_df = grouped_df.agg({"col_2": "nunique"})

# check for NaN under a single DataFrame column:
df['col_1'].isnull().values.any()

# count the NaN under a single DataFrame column:
# False -> 0, True -> 1
df['col_1'].isnull().sum()

# check for NaN under an entire DataFrame:
df.isnull().values.any()

# only view missing values
df[df["col_1"].isnull()]

# count the NaN under an entire DataFrame:
df.isnull().sum()  # sum of each row
df.isnull().sum().sum()

Using operator itemgetter:

>>> s = pd.Series([(1,"a"), (2,"b")], name="col1")
>>> df = s.to_frame()
>>>df["col1"].apply(itemgetter(0))
0    1
1    2
Name: col1, dtype: int64

>>>df["col1"].apply(itemgetter(1))
0    a
1    b
Name: col1, dtype: object

That definitely works! You can also use the `.str` accessor to do this too:

>>> s = pd.Series([(1,"a"), (2,"b")])
>>> s.str.get(0)
0 1
1 2
dtype: int64

>>> s.str.get(1)
0 a
1 b
dtype: object

Alternatively, you can explode our these values to work with them in a different way.
>>> s.explode()
0 1
0 a
1 2
1 b
dtype: object

df["col_3"] = df["col_1"] + " " + df["col2"]
df[["col_1", "col_2"]] = df["col_3"].str.split(" ", expand=True)

import pandas as pd 

# Specify column names separately 
column_names = ['A', 'B', 'C']  

# Data as a list of lists 
data = [     
  [1, 2, 3],     
  [4, 5, 6],     
  [7, 8, 9] 
]  

# Create DataFrame 
df = pd.DataFrame(data, columns=column_names)  

print(df)