3 Sun
TIL
[인프런] 단 두 장의 문서로 데이터 분석과 시각화 뽀개기
- Summarize Data
import pandas as pd
import seaborn as sns
import numpy as npdf = sns.load_dataset('iris')
df.shape(150, 5)df.head(2)sepal_length
sepal_width
petal_length
petal_width
species
0
5.1
3.5
1.4
0.2
setosa
1
4.9
3.0
1.4
0.2
setosa
df['w'].value_counts()
해당 키의 원소의 개수를 출력한다. 종류의 개수를 확인할 때 유용하다.
df['species'].value_counts()setosa 50
virginica 50
versicolor 50
Name: species, dtype: int64df['petal_width'].value_counts()0.2 29
1.3 13
1.5 12
1.8 12
1.4 8
2.3 8
1.0 7
0.3 7
0.4 7
2.0 6
2.1 6
0.1 5
1.2 5
1.9 5
1.6 4
2.5 3
2.2 3
2.4 3
1.1 3
1.7 2
0.6 1
0.5 1
Name: petal_width, dtype: int64pd.DataFrame(df['species'].value_counts())species
setosa
50
virginica
50
versicolor
50
len(df)
df의 길이를 알 수 있다. shape로도 알 수 있음
len(df)150df.shape[0], df.shape[1](150, 5)len(df) == df.shape[0]Truedf['w'].nunique()
고유값의 개수를 보여준다
df['species'].nunique()3df['sepal_width'].nunique()23df.describe()
수치형 데이터들에 대한 통계값을 보여준다. [option] include : 포함, exclude : 제외 all : 데이터 타입과 관련없이 모든 데이터 np.number : numpy로 숫자 형태의 데이터 np.object : object 타입의 데이터 category : category 타입의 데이터
df.describe()sepal_length
sepal_width
petal_length
petal_width
count
150.000000
150.000000
150.000000
150.000000
mean
5.843333
3.057333
3.758000
1.199333
std
0.828066
0.435866
1.765298
0.762238
min
4.300000
2.000000
1.000000
0.100000
25%
5.100000
2.800000
1.600000
0.300000
50%
5.800000
3.000000
4.350000
1.300000
75%
6.400000
3.300000
5.100000
1.800000
max
7.900000
4.400000
6.900000
2.500000
df.describe(include='all')sepal_length
sepal_width
petal_length
petal_width
species
count
150.000000
150.000000
150.000000
150.000000
150
unique
NaN
NaN
NaN
NaN
3
top
NaN
NaN
NaN
NaN
setosa
freq
NaN
NaN
NaN
NaN
50
mean
5.843333
3.057333
3.758000
1.199333
NaN
std
0.828066
0.435866
1.765298
0.762238
NaN
min
4.300000
2.000000
1.000000
0.100000
NaN
25%
5.100000
2.800000
1.600000
0.300000
NaN
50%
5.800000
3.000000
4.350000
1.300000
NaN
75%
6.400000
3.300000
5.100000
1.800000
NaN
max
7.900000
4.400000
6.900000
2.500000
NaN
df.describe(include=[np.object])species
count
150
unique
3
top
setosa
freq
50
setosa 종이 가장 많이 등장하는 종 중에서 하나인데 50번 등장한다.
some functions()
sum()
Sum values of each object.
count()
Count non-NA/null values of
each object.
median()
Median value of each object.
quantile([0.25,0.75])
Quantiles of each object.
apply(function)
Apply function to each object.
min()
Minimum value in each object.
max()
Maximum value in each object.
mean()
Mean value of each object.
var()
Variance of each object.
std()
Standard deviation of each
object.# 총합
df['petal_width'].sum()179.90000000000003# 갯수
df['petal_width'].count()150# 중간값
df['petal_width'].median()1.3# 평균값
df['petal_width'].mean()1.1993333333333336# 사분위값
df.quantile([0.25, 0.75])sepal_length
sepal_width
petal_length
petal_width
0.25
5.1
2.8
1.6
0.3
0.75
6.4
3.3
5.1
1.8
# 분산
df.var()sepal_length 0.685694
sepal_width 0.189979
petal_length 3.116278
petal_width 0.581006
dtype: float64# 표준편차
df.std()sepal_length 0.828066
sepal_width 0.435866
petal_length 1.765298
petal_width 0.762238
dtype: float64판다스로 apply 활용하기 lambda 익명함수 사용하기 - Summarize Data
df.apply()
df.apply(lambda x : x[2])sepal_length 4.7
sepal_width 3.2
petal_length 1.3
petal_width 0.2
species setosa
dtype: objectdf['species_3'] = df['species'].apply(lambda x : x[:3])
dfsepal_length
sepal_width
petal_length
petal_width
species
species_3
0
5.1
3.5
1.4
0.2
setosa
set
1
4.9
3.0
1.4
0.2
setosa
set
2
4.7
3.2
1.3
0.2
setosa
set
3
4.6
3.1
1.5
0.2
setosa
set
4
5.0
3.6
1.4
0.2
setosa
set
...
...
...
...
...
...
...
145
6.7
3.0
5.2
2.3
virginica
vir
146
6.3
2.5
5.0
1.9
virginica
vir
147
6.5
3.0
5.2
2.0
virginica
vir
148
6.2
3.4
5.4
2.3
virginica
vir
149
5.9
3.0
5.1
1.8
virginica
vir
150 rows × 6 columns
# 뒤에서 3번째 까지의 문자를 가져오는 함수
def smp(x):
x = x[-3:]
return xdf['species-3'] = df['species'].apply(smp)
dfsepal_length
sepal_width
petal_length
petal_width
species
species_3
species-3
0
5.1
3.5
1.4
0.2
setosa
set
osa
1
4.9
3.0
1.4
0.2
setosa
set
osa
2
4.7
3.2
1.3
0.2
setosa
set
osa
3
4.6
3.1
1.5
0.2
setosa
set
osa
4
5.0
3.6
1.4
0.2
setosa
set
osa
...
...
...
...
...
...
...
...
145
6.7
3.0
5.2
2.3
virginica
vir
ica
146
6.3
2.5
5.0
1.9
virginica
vir
ica
147
6.5
3.0
5.2
2.0
virginica
vir
ica
148
6.2
3.4
5.4
2.3
virginica
vir
ica
149
5.9
3.0
5.1
1.8
virginica
vir
ica
150 rows × 7 columns
fillna, dropna로 결측치 다루기 - Handling Missing Data
결측치(Not a ...) 에 대해서 처리하는 방법
df.dropna()
Drop rows with any column having NA/null data.
df.fillna(value)
Replace all NA/null data with value.df = pd.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],
"toy": [np.nan, 'Batmobile', 'Bullwhip'],
"born": [pd.NaT, pd.Timestamp("1940-04-25"),pd.NaT]})
dfname
toy
born
0
Alfred
NaN
NaT
1
Batman
Batmobile
1940-04-25
2
Catwoman
Bullwhip
NaT
df.dropna?# axis = 0 : 행, axis = 1 : 열
# how = 'all' : 모두 널 값일 때, how = 'any' : 하나라도 널 값일 때
df.dropna(axis=1, how='all')name
toy
born
0
Alfred
NaN
NaT
1
Batman
Batmobile
1940-04-25
2
Catwoman
Bullwhip
NaT
df.dropna(axis=1, how='any')name
0
Alfred
1
Batman
2
Catwoman
df.dropna(axis=0, how='any')name
toy
born
1
Batman
Batmobile
1940-04-25
df.fillna?df = pd.DataFrame([[np.nan, 2, np.nan, 0],
[3, 4, np.nan, 1],
[np.nan, np.nan, np.nan, 5],
[np.nan, 3, np.nan, 4]],
columns=list('ABCD'))
dfA
B
C
D
0
NaN
2.0
NaN
0
1
3.0
4.0
NaN
1
2
NaN
NaN
NaN
5
3
NaN
3.0
NaN
4
values = {'A' : 0, 'B' : 1, 'C' : 2, 'D' : 3}
df.fillna(value=values)A
B
C
D
0
0.0
2.0
2.0
0
1
3.0
4.0
2.0
1
2
0.0
1.0
2.0
5
3
0.0
3.0
2.0
4
df.fillna(df['D'].mean())A
B
C
D
0
2.5
2.0
2.5
0
1
3.0
4.0
2.5
1
2
2.5
2.5
2.5
5
3
2.5
3.0
2.5
4
df.isnull()A
B
C
D
0
True
False
True
False
1
False
False
True
False
2
True
True
True
False
3
True
False
True
False
df.isnull().sum()A 3
B 1
C 4
D 0
dtype: int64df.notnull().sum()A 1
B 3
C 0
D 4
dtype: int64assign 으로 새로운 컬럼 만들기, qcut으로 binning, bucketing 하기 - Make New Columns
df = pd.DataFrame({'A': range(1, 11),
'B' : np.random.randn(10)})
dfA
B
0
1
0.052204
1
2
-1.489858
2
3
0.427285
3
4
1.148815
4
5
-1.301116
5
6
1.739656
6
7
1.000600
7
8
-1.672363
8
9
0.301468
9
10
-0.221703
df.assign(Area=lambda df: df.Length*df.Height)
Compute and append one or more new columns.
df['Volume'] = df.Length*df.Height*df.Depth
Add single column.df.assign?df.assign(ln_A = lambda x: np.log(x.A)).head()A
B
ln_A
0
1
0.052204
0.000000
1
2
-1.489858
0.693147
2
3
0.427285
1.098612
3
4
1.148815
1.386294
4
5
-1.301116
1.609438
df['ln_A'] = np.log(df.A).head()
df.head()A
B
ln_A
0
1
0.052204
0.000000
1
2
-1.489858
0.693147
2
3
0.427285
1.098612
3
4
1.148815
1.386294
4
5
-1.301116
1.609438
assign을 통해서 새로운 컬럼을 만들거나, 직접 할당을 해서 만들 수 있다
pd.qcut(df.col, n, labels=False)
Bin column into n buckets.n개의 버킷 수 만큼 새로운 컬럼을 만들으라는 의미
pd.qcut?pd.qcut(range(5), 4)[(-0.001, 1.0], (-0.001, 1.0], (1.0, 2.0], (2.0, 3.0], (3.0, 4.0]]
Categories (4, interval[float64]): [(-0.001, 1.0] < (1.0, 2.0] < (2.0, 3.0] < (3.0, 4.0]]pd.qcut(range(5), 3, labels=["good", "medium", "bad"])['good', 'good', 'medium', 'bad', 'bad']
Categories (3, object): ['good' < 'medium' < 'bad']pd.qcut(df.B, 3, labels=["good", "medium", "bad"])0 medium
1 good
2 medium
3 bad
4 good
5 bad
6 bad
7 good
8 medium
9 good
Name: B, dtype: category
Categories (3, object): ['good' < 'medium' < 'bad']pd.qcut(df.B, 2, labels=["good", "bad"])0 good
1 good
2 bad
3 bad
4 good
5 bad
6 bad
7 good
8 bad
9 good
Name: B, dtype: category
Categories (2, object): ['good' < 'bad']max(axis=1)
Element-wise max.
clip(lower=-10,upper=10)
Trim values at input thresholds
min(axis=1)
Element-wise min.
abs()
Absolute value.df.max(axis = 0)A 10.000000
B 1.739656
ln_A 1.609438
dtype: float64df.max(axis = 1)0 1.0
1 2.0
2 3.0
3 4.0
4 5.0
5 6.0
6 7.0
7 8.0
8 9.0
9 10.0
dtype: float64df.min(axis = 0)A 1.000000
B -1.672363
ln_A 0.000000
dtype: float64df.max(axis = 1)0 1.0
1 2.0
2 3.0
3 4.0
4 5.0
5 6.0
6 7.0
7 8.0
8 9.0
9 10.0
dtype: float64df['A'].clip(lower=-10,upper=10)0 1
1 2
2 3
3 4
4 5
5 6
6 7
7 8
8 9
9 10
Name: A, dtype: int64df['A'].clip(lower=2,upper=8)0 2
1 2
2 3
3 4
4 5
5 6
6 7
7 8
8 8
9 8
Name: A, dtype: int64df['B'].clip(lower=1,upper=1.5)0 1.000000
1 1.000000
2 1.000000
3 1.148815
4 1.000000
5 1.500000
6 1.000600
7 1.000000
8 1.000000
9 1.000000
Name: B, dtype: float64df["B"]0 0.052204
1 -1.489858
2 0.427285
3 1.148815
4 -1.301116
5 1.739656
6 1.000600
7 -1.672363
8 0.301468
9 -0.221703
Name: B, dtype: float64df["B"].abs()0 0.052204
1 1.489858
2 0.427285
3 1.148815
4 1.301116
5 1.739656
6 1.000600
7 1.672363
8 0.301468
9 0.221703
Name: B, dtype: float64Last updated
Was this helpful?