Python數據科學入門（Pandas玩轉數據）

慕課網數據科學入門課程學習筆記

一、Series和DataFrame的簡單數學運算

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

1.Series 相加

s1 = Series([1,2,3],index=['A','B','C'])
s1

A 1 B 2 C 3 dtype: int64

s2 = Series([4,5,6,7],index=['B','C','D','E'])
s2

B 4 C 5 D 6 E 7 dtype: int64

# 對應的 index 的值相加，沒有 nan + 非 nan還是nan
s1 + s2     

A NaN B 6.0 C 8.0 D NaN E NaN dtype: float64

2.Dataframe運算

df1 = DataFrame(np.arange(4).reshape(2,2),index=['A','B'],columns=['AA','BB'])
df1

df2 = DataFrame(np.arange(9).reshape(3,3),index=['A','B','C'],columns=['AA','BB','CC'])
df2

#加法運算
df1 + df2

df3 = DataFrame([[1,2,3],[4,5,np.nan],[7,8,9]],
                index=['A','B','C'],columns=['c1','c2','c3'])
df3

# 默認列和    這里的 nan 被忽略
# axis = 1 時求 行和
df3.sum()

c1 12.0 c2 15.0 c3 12.0 dtype: float64

# min() max()  默認求列
df3.min()

c1 1.0 c2 2.0 c3 3.0 dtype: float64

df3.describe()

---

二、Series和DataFrame的排序

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

1.Series排序

s1 = Series(np.random.randn(10))
s1

0 0.773184 1 -0.403266 2 0.980296 3 -0.631971 4 0.636405 5 0.732921 6 0.756932 7 -0.597943 8 -1.152224 9 0.771753 dtype: float64

# 按照values 排序 默認升序
# ascending=False  降序
s1.sort_values()

8 -1.152224 3 -0.631971 7 -0.597943 1 -0.403266 4 0.636405 5 0.732921 6 0.756932 9 0.771753 0 0.773184 2 0.980296 dtype: float64

# 按照index 排序 默認升序
# ascending=False  降序
s1.sort_index(ascending=False)

9 0.771753 8 -1.152224 7 -0.597943 6 0.756932 5 0.732921 4 0.636405 3 -0.631971 2 0.980296 1 -0.403266 0 0.773184 dtype: float64

2.DataFrame排序

df1 = DataFrame(np.random.randn(20).reshape(4,5),columns=['a','b','c','d','e'])
df1

#   按照 a 列排序
# 默認 升序  ascending=False 降序
# 默認 axis=0 針對列 
df1.sort_values('a',ascending=False)

---

三、重命名DataFrame的index

import numpy as np
import pandas as pa
from pandas import Series,DataFrame

1.重命名 index

df1 = DataFrame(np.arange(9).reshape(3,3),index=['A','B','C'],columns=['AA','BB','CC'])
df1

#重新給index賦值
df1.index=Series(['a','b','c'])
df1

#  map 方法
df1.index = df1.index.map(str.upper)
df1

# rename 方法
df1.rename(index=str.lower,columns=str.lower)

df1

# 通過字典 對特定的index修改
# 更多 查看文檔
df1.rename(index={'A':'g'})

2.復習

list1 = [1,2,3,4]
list2 = ['1','2','3','4']

# 將 list1 元素轉換為 字符串
[str(x) for x in list1]

[‘1’, ‘2’, ‘3’, ‘4’]

# map 方法轉換
list(map(str,list1))

[‘1’, ‘2’, ‘3’, ‘4’]

3.寫一個自己的 map

def test_map(x):
    return x + '_ABC'
df1.index.map(test_map)

Index([‘A_ABC’, ‘B_ABC’, ‘C_ABC’], dtype=’object’)

df1.rename(index=test_map)

---

四、DataFrame 的merge（合并）操作

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

df1 = DataFrame({'key':['X','Y','Z'],'data_set_1':[1,2,3]})
df1

df2 = DataFrame({'key':['X','X','C'],'data_set_2':[4,5,6]})
df2

# 默認會去找 兩個 df 都有的 key
# 可通過 on = ？ 指定
# how=inner/left/outer/right  等參數查看文檔
#   df1 和 df2 都有 key
pd.merge(df1,df2)

# left 去掉右邊df2 “多余”
pd.merge(df1,df2,on='key',how='left')

# right 去掉左邊df1 “多余”
pd.merge(df1,df2,on='key',how='right')

# outer 像是 left 和 right 的結合  都保留
pd.merge(df1,df2,on='key',how='outer')

---

五、Concatenaten和Combine

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

1>Concatenaten

1.Numpy array

arr1 = np.arange(9).reshape(3,3)
arr1

array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])

arr2 = np.arange(9).reshape(3,3)
arr2

array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])

# concatenate 默認 axis=0 豎著連接
np.concatenate([arr1,arr2])

array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [0, 1, 2], [3, 4, 5], [6, 7, 8]])

2.Series

s1 = Series([1,2,3],index=['X','Y','Z'])
s2 = Series([4,5],index=['A','B'])

pd.concat([s1,s2])

X 1 Y 2 Z 3 A 4 B 5 dtype: int64

pd.concat([s1,s2],axis=1)

3.DataFrame

df1 = DataFrame(np.random.randn(4,3),columns=['X','Y,','Z'])
df1

df2 = DataFrame(np.random.randn(3,3),columns=['X','Y,','A'])
df2

pd.concat([df1,df2])

	A	X	Y,	Z
0	NaN	0.542921	1.710888	1.027810
1	NaN	0.624385	-0.825916	0.893589
2	NaN	-1.000241	2.551461	1.541975
3	NaN	0.020806	0.635956	0.573629
0	-1.701335	-0.721702	-0.106360	NaN
1	0.212189	0.591009	0.233578	NaN
2	0.247578	0.855895	-1.831944	NaN

2> Combine

1.Series

s1 = Series([2,np.nan,4,np.nan],index=['A','B','C','D'])
s1

A 2.0 B NaN C 4.0 D NaN dtype: float64

s2 = Series([1,2,3,4],index=['A','B','C','D'])
s2

A 1 B 2 C 3 D 4 dtype: int64

# s1 中的 nan 被 s2 填充
s1.combine_first(s2)

A 2.0 B 2.0 C 4.0 D 4.0 dtype: float64

2.DataFrame

df1 = DataFrame({'X':[1, np.nan, 3, np.nan],
                'Y':[5, np.nan, 7, np.nan],
                'Z':[9, np.nan, 11, np.nan]})
df1

df2 = DataFrame({'Z':[np.nan, 10, np.nan, 12],
                'A':[1,2,3,4],})
df2

# df1 里的 nan 被 df2 填充
df1.combine_first(df2)

---

六、通過apply進行數據預處理

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

df = pd.read_csv('data.csv')
df.head()

df.size

190

s1 = Series(['a']*190)

df['A'] = s1
df.head()

# apply() 對一列操作
df['A'] = df['A'].apply(str.upper)
df.head()

將data分割多列

# 以空格分割  
df['data'][0].split(' ')

[”, ‘AA:’, ‘BB:’, ‘CC:’, ‘DD’]

# 去掉空格干擾
list_1 = df['data'][0].strip().split(' ')
list_1

[‘AA:’, ‘BB:’, ‘CC:’, ‘DD’]

要求：將第2列和第4列取出重新放入data列

使用 head() 需要注意，如果對 df 操作時使用1了 head()會使只操作了前幾行

使用 head() 來查看就好

list_1[1],list_1[3]

(‘BB:’, ‘DD’)

def foo(line):

    items = line.strip().split(' ')
    return Series([items[1],items[3]])

df_tmp = df['data'].apply(foo)

df_tmp.head()

df_tmp = df_tmp.rename(columns={0:"B",1:"D"})
df_tmp.head()

df.head()

# 連接操作
df_new = df.combine_first(df_tmp)
df_new.head()

# 刪除一些不需要的列
del df_new['A']
del df_new['data']
df_new.head()

# 存儲 df_new.csv
df_new.to_csv('df_new.csv')

——–

七、通過去重清洗數據

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

df = pd.read_csv('data.csv')

df.head()

df.size

190

len(df)

95

# 查看 time 列 不重復的數據 的數量
len(df['time'].unique())

22

去重

# 判重 的方法
df['time'].duplicated().head()

0 False 1 True 2 True 3 False 4 True Name: time, dtype: bool

# 對 duplicated() 方法返回 True 的數據刪掉，達到去重
df_new = df.drop_duplicates()
df_new.head()

#  以 data 列為標準來刪除重復,keep='last' 表示保存最后一個
df_new = df.drop_duplicates({'data'},keep='last')
df_new.head()

---

八、時間序列的操作基礎

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

from datetime import datetime
t1 = datetime(2009,10,20)
t1

datetime.datetime(2009, 10, 20, 0, 0)

date_list = [
    datetime(2016,9,1),
    datetime(2016,9,10),
    datetime(2017,9,1),
    datetime(2017,9,10),
    datetime(2017,10,1)
]
date_list

[datetime.datetime(2016, 9, 1, 0, 0), datetime.datetime(2016, 9, 10, 0, 0), datetime.datetime(2017, 9, 1, 0, 0), datetime.datetime(2017, 9, 10, 0, 0), datetime.datetime(2017, 10, 1, 0, 0)]

s1 = Series(np.random.rand(5),index=date_list)
s1

2016-09-01 0.497374 2016-09-10 0.686276 2017-09-01 0.788562 2017-09-10 0.765383 2017-10-01 0.546197 dtype: float64

s1[1]

0.6862758320157275

s1[datetime(2016,9,10)]

0.6862758320157275

s1['2016-9-10']

0.6862758320157275

s1['20160910']

0.6862758320157275

s1['2016-9']

2016-09-01 0.497374 2016-09-10 0.686276 dtype: float64

s1['2017']

2017-09-01 0.788562 2017-09-10 0.765383 2017-10-01 0.546197 dtype: float64

# date_range(start,end,periods...)  隨機生成時間 
# freq = 'D/W/'  D 間隔為 天   
# W間隔為周，W-MON 從周一開始（默認從周日開始）默認 D
# H 間隔為 小時  5H 間隔為 5小時
date_list_new = pd.date_range(start='2017-01-01',periods=10)
date_list_new

DatetimeIndex([‘2017-01-01’, ‘2017-01-02’, ‘2017-01-03’, ‘2017-01-04…)

s2 = Series(np.random.rand(10),index=date_list_new)
s2

2017-01-01 0.152626 2017-01-02 0.590219 … Freq: D, dtype: float64 ——

九、時間序列數據的采樣和畫圖

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

t_range = pd.date_range('2016-01-01','2016-12-01')
t_range

DatetimeIndex([‘2016-01-01’, ‘2016-01-02’, ‘2016-01-03’, ‘2016-01-04’, ‘2016-01-05’, ‘2016-01-06’, ‘2016-01-07’, ‘2016-01- ….)

s1 = Series(np.random.randn(len(t_range)),index=t_range)
s1.head()

2016-01-01    0.145550
2016-01-02    2.444428
...
Freq: D, dtype: float64

采樣

#  t_range 是每天一個數據，下面要求每月一個數據
# 要求 計算每個月的 平均值 
s1['2016-01'].mean()

-0.14446619115276657

# shift + tab 鍵查看方法說明
# M 月 H 小時
s1_month = s1.resample('M').mean()
s1_month.head()

2016-01-31   -0.144466
2016-02-29    0.083245
2016-03-31    0.204149

Freq: M, dtype: float64

數據填充

# ffill() 往前填充
s1_test = s1.resample('H').ffill()
s1_test.head()

2016-01-01 00:00:00 0.14555 2016-01-01 01:00:00 0.14555 2016-01-01 02:00:00 0.14555 2016-01-01 03:00:00 0.14555 2016-01-01 04:00:00 0.14555 Freq: H, dtype: float64

# bfill() 往后填充
s1_test = s1.resample('H').bfill()
s1_test.head()

2016-01-01 00:00:00    0.145550
2016-01-01 01:00:00    2.444428
2016-01-01 02:00:00    2.444428
2016-01-01 03:00:00    2.444428
2016-01-01 04:00:00    2.444428
Freq: H, dtype: float64

簡單畫圖

t_range = pd.date_range('2016-01-01','2016-12-31',freq='H')
t_range

DatetimeIndex(['2016-01-01 00:00:00', '2016-01-01 01:00:00',
               '2016-01-01 02:00:00', '2016-01-01 03:00:00',
               '2016-01-01 04:00:00', '2016-01-01 05:00:00',
               '2016-01-01 06:00:00', '2016-01-01 07:00:00',
               '2016-01-01 08:00:00', '2016-01-01 09:00:00',
               ...
         )

stock_df = DataFrame(index=t_range)
stock_df.head(10)

stock_df['data1'] = np.random.randint(80,160,size=len(t_range))
stock_df.head(10)

stock_df['data2'] = np.random.randint(40,140,size=len(t_range))
stock_df.head(10)

stock_df.plot()

<matplotlib.axes._subplots.AxesSubplot at 0x249d416d438>

import matplotlib.pyplot as plt
plt.show()

按周取樣再繪圖

weekly_df = DataFrame()
weekly_df['data1'] = stock_df['data1'].resample('W').mean()
weekly_df['data2'] = stock_df['data2'].resample('W').mean()

weekly_df.head()

weekly_df.plot()
plt.show()

---

十、數據分箱技術-Binning

什么是數據分箱？

例如 不同大小的蘋果分裝在不同的箱子

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

score_list = np.random.randint(0,100,size=20)
score_list

array([87, 83, 40, 14, 4, 31, 46, 17, 96, 99, 39, 4, 85, 8, 64, 82, 32, 28, 36, 40])

bins = [0,59,70,80,100]

數據分箱

# cut(array,list)  將array里的數據按list區間劃分 
score_cut = pd.cut(score_list,bins)
score_cut

[(80, 100], (80, 100], (0, 59], (0, 59], (0, 59], …, (80, 100], (0, 59], (0, 59], (0, 59], (0, 59]] Length: 20 Categories (4, interval[int64]): [(0, 59]

# value_counts() 統計各個 數值區間的 個數
pd.value_counts(score_cut)

(0, 59] 13 (80, 100] 6 (59, 70] 1 (70, 80] 0 dtype: int64

df = DataFrame()

df['score'] = score_list
df['student'] = [pd.util.testing.rands(3) for i in range(20)]
df.head()

數據分箱 cut()函數

cut(數據,區間-list，labels=[‘各區間的表示標簽’,’…’])

df['Categories'] = pd.cut(df['score'],bins,labels=['Low','OK','Good','Great'])
df.head()

---

十一、數據分組技術-GroupBy

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

原理類似SQL

df = pd.read_csv('weather.csv')
df.head()

分組

g = df.groupby(df['city'])
g

# 查看分組情況  哪一列 哪些組
g.groups

{‘BJ’: Int64Index([1, 4, 7, 10, 13, 16, 19, 22, 25, 28], dtype=’int64’), ‘GZ’: Int64Index([0, 3, 6, 9, 12, 15, 18, 21, 24, 27], dtype=’int64’), ‘SH’: Int64Index([2, 5, 8, 11, 14, 17, 20, 23, 26], dtype=’int64’)}

#查看某組
g_gz = g.get_group('GZ')
g_gz.head()

# 求gz的平均值
g_gz.mean()

temperature 9.0 wind 4.5 dtype: float64

# 求整個組的平均值
g.mean()

---

十二、數據聚合技術Aggregation

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

df = pd.read_csv('weather.csv')
g = df.groupby('city')

聚合

mean() count() max() 等都是聚合

# g.agg('min')  就是 g.min()  其他類似
g.agg('min')

自定義函數 來聚合

def foo(attr):
    return attr.max() - attr.min()

g.agg(foo)

多columns分組

g_new = df.groupby(['city','wind'])
g_new.groups

{(‘BJ’, 1): Int64Index([13], dtype=’int64’), (‘BJ’, 2): Int64Index([25], dtype=’int64’), (‘BJ’, 3): Int64Index([4, 10], dtype=’int64’), (‘BJ’, 4): Int64Index([7], dtype=’int64’), ……}

g_new.get_group(('BJ',3))

對多 columns 遍歷需要注意

for (name_1,name_2),group in g_new:
    print(name_1,name_2)
    print(group)

BJ 1
        date city  temperature  wind
13  2017/4/2   BJ            2     1
BJ 2
         date city  temperature  wind
25  2017/6/25   BJ            2     2

…..

---

十三、透視表

透視表概念來自Excel,為了更好展示數據，對原來的行列進行了一些變化，根據這些變化，相應的數據也變化

import numpy as np
import pandas as pd
from pandas import Series,DataFrame

無數據源 不演示了

# 讀取excel文件數據
df = pd.read_excel()
# 生成數據透視表,index 對哪一列進行透視
# values 接收一些想要說的數據
pd.pivot_table(df,index=[''],...)