Pandas - 图1
© getcodify.com

由於語法渲染問題而影響閱讀體驗， 請移步博客閱讀～
本文GitPage地址

Pandas

Basic

  1. import pandas as pd
  3. ###read file
  5. TB = pd.read_csv("sentence-G",sep='\t', names=None, header=None)
  6. TB.columns = ['频率', '', 'V2', '词义', 'V3']
  8. ##DataFrame
  9. pd.DataFrame(index=range(40),columns=['a', 'b'])
  11. ### rsult output
  12. TB.to_csv("table",sep='\t')
  14. data
  15. data[0:2]       #取前两行数据
  17. len(data )              #求出一共多少行
  18. data.columns.size      #求出一共多少列
  19. data.columns        #列索引名称
  20. data.index       #行索引名称
  22. data.ix[1]                #取第2行数据
  23. data.iloc[1]             #取第2行数据
  24. data.loc['A']      #取第行索引为”A“的一行数据，
  25. data['x']      #取列索引为x的一列数据
  27. ## Most frequent value
  28. data.mode()
  30. ## Any True value in this series
  31. data['A'].any()
  33. ## All the values are True
  34. data['A'].all()
  36. data.loc[:,['x','z'] ]          #表示选取所有的行以及columns为a,b的列；
  37. data.loc[['A','B'],['x','z']]     #表示选取'A'和'B'这两行以及columns为x,z的列的并集；
  38. data.iloc[1:3,1:3]              #数据切片操作，切连续的数据块
  39. data.iloc[[0,2],[1,2]]              #即可以自由选取行位置，和列位置对应的数据，切零散的数据块
  41. data[data>2]       #表示选取数据集中大于0的数据
  42. data[data.x>5]       #表示选取数据集中x这一列大于5的所有的行
  43. a1=data.copy()
  44. a1[a1['y'].isin(['6','10'])]    #表显示满足条件：列y中的值包含'6','8'的所有行。
  46. data.mean()           #默认对每一列的数据求平均值；若加上参数a.mean(1)则对每一行求平均值；
  47. data['x'].value_counts()    #统计某一列x中各个值出现的次数：
  48. data.describe() #对每一列数据进行统计，包括计数，均值，std，各个分位数等。
  49. data.to_excel(r'E:\pypractice\Yun\doc\2.xls',sheet_name='Sheet1')  #数据输出至Exceldata[0:2]       #取前两行数据
  51. ### from Dictionary to DataFrame
  52. TB = pd.Series(BB)
  54. ### DataFrame sort
  55. TB = TB.sort_values(ascending=False)
  57. ### DataFrame merge
  58. result = pd.concat([ Word, Sen], axis=1, sort=False)
  60. ### NaN drap
  61. data.dropna(thresh=3) # at least 3 data we have

Skills

reference: 数据分析1480

NA count

  1. df=pd.read_csv('titanic_train.csv')
  2. def missing_cal(df):
  3.     """
  4.     df :数据集  
  5.     return：每个变量的缺失率
  6.     """
  7.     missing_series = df.isnull().sum()/df.shape[0]
  8.     missing_df = pd.DataFrame(missing_series).reset_index()
  9.     missing_df = missing_df.rename(columns={'index':'col',
  10.                                             0:'missing_pct'})
  11.     missing_df = missing_df.sort_values('missing_pct',ascending=False).reset_index(drop=True)
  12.     return missing_df
  13. missing_cal(df)

Drop/Fill NA

  1. ## Drop the columns or rows by NA value
  2. data.dropna(axis=0)
  3. data.dropna(axis=1)
  5. ## Fill the NA value by linear interpolation
  6. data.interpolate()
  8. ## Fill te NA value with the value ahead or the next
  9. data.fillna(method='ffill')
  10. data.fillna(method='backfill')

idmax

  1. df = pd.DataFrame({'Sp':['a','b','c','d','e','f'], 'Mt':['s1', 's1', 's2','s2','s2','s3'], 'Value':[1,2,3,4,5,6], 'Count':[3,2,5,10,10,6]})
  2. df
  4. df.iloc[df.groupby(['Mt']).apply(lambda x: x['Count'].idxmax())]
  6. df["rank"] = df.groupby("ID")["score"].rank(method="min", ascending=False).astype(np.int64)
  7. df[df["rank"] == 1][["ID", "class"]]

Merging DataFrame

This is one of the most feature I like in pandas since it could automatically fill the missing value with NA.
Plus, when the DataFrame goes huge, pd.concat was way faster than dataframe merge in R.

  1. df = pd.DataFrame({'id_part':['a','b','c','d'], 'pred':[0.1,0.2,0.3,0.4], 'pred_class':['women','man','cat','dog'], 'v_id':['d1','d2','d3','d1']})
  3. ##  Row
  4. pd.concat([df,df], axis=1)
  5. ## or
  6. df.merge(df)
  8. ## Column
  9. pd.concat([df,df])
  10. ## or
  11. df.append(df)
  13. ## I forget what the codes here for = =
  14. ##df.groupby(['v_id']).agg({'pred_class': [', '.join],'pred': lambda x: list(x),
  15. ##    'id_part': 'first'}).reset_index()

Deleting rows by string-match

  1. df = pd.DataFrame({'a':[1,2,3,4], 'b':['s1', 'exp_s2', 's3','exps4'], 'c':[5,6,7,8], 'd':[3,2,5,10]})
  2. df[df['b'].str.contains('exp')]

Sort

  1. df = pd.DataFrame([['A',1],['A',3],['A',2],['B',5],['B',9]], columns = ['name','score'])
  3. df.sort_values(['name','score'], ascending = [True,False])
  4. df.groupby('name').apply(lambda x: x.sort_values('score', ascending=False)).reset_index(drop=True)

Select columns by features

  1. drinks = pd.read_csv('data/drinks.csv')
  2. ## 选择所有数值型的列
  3. drinks.select_dtypes(include=['number']).head()
  4. ## 选择所有字符型的列
  5. drinks.select_dtypes(include=['object']).head()
  6. drinks.select_dtypes(include=['number','object','category','datetime']).head()
  7. ## 用 exclude 关键字排除指定的数据类型
  8. drinks.select_dtypes(exclude=['number']).head()

str to integer

  1. df = pd.DataFrame({'列1':['1.1','2.2','3.3'],
  2.                   '列2':['4.4','5.5','6.6'],
  3.                   '列3':['7.7','8.8','-']})
  4. df
  5. df.astype({'列1':'float','列2':'float'}).dtypes
  8. df = df.apply(pd.to_numeric, errors='coerce').fillna(0)

Reduce the RAM-consume

  1. cols = ['beer_servings','continent']
  2. small_drinks = pd.read_csv('data/drinks.csv', usecols=cols)
  1. dtypes ={'continent':'category'}
  2. smaller_drinks = pd.read_csv('data/drinks.csv',usecols=cols, dtype=dtypes)

根据最大的类别筛选 DataFrame

  1. movies = pd.read_csv('data/imdb_1000.csv')
  2. counts = movies.genre.value_counts()
  3. movies[movies.genre.isin(counts.nlargest(3).index)].head()

split string to columns

  1. df = pd.DataFrame({'姓名':['张 三','李 四','王 五'],
  2.                    '所在地':['北京-东城区','上海-黄浦区','广州-白云区']})
  3. df
  4. df.姓名.str.split(' ', expand=True)

str.contain

  1. df.['column1'].str.cotain('A')

把 Series 里的列表转换为 DataFrame

  1. df = pd.DataFrame({'列1':['a','b','c'],'列2':[[10,20], [20,30], [30,40]]})
  3. pd.concat([df,df_new], axis='columns')

用多个函数聚合

  1. orders = pd.read_csv('data/chipotle.tsv', sep='\t')
  2. orders.groupby('order_id').item_price.agg(['sum','count']).head()

分组聚合

  1. import pandas as pd
  2. df = pd.DataFrame({'key1':['a', 'a', 'b', 'b', 'a'],
  3.     'key2':['one', 'two', 'one', 'two', 'one'],
  4.     'data1':np.random.randn(5),
  5.      'data2':np.random.randn(5)})
  6. df
  8. for name, group in df.groupby('key1'):
  9.     print(name)
  10.     print(group)
  12. dict(list(df.groupby('key1')))

通过字典或Series进行分组

  1. people = pd.DataFrame(np.random.randn(5, 5),
  2.      columns=['a', 'b', 'c', 'd', 'e'],
  3.      index=['Joe', 'Steve', 'Wes', 'Jim', 'Travis'])
  4. mapping = {'a':'red', 'b':'red', 'c':'blue',
  5.      'd':'blue', 'e':'red', 'f':'orange'}
  6. by_column = people.groupby(mapping, axis=1)
  7. by_column.sum()

Connect to the matplotlib

  1. import numpy as np
  2. import pandas as pd
  3. from matplotlib import pyplot as plt
  6. dates=pd.date_range('20180310',periods=6)
  7. df = pd.DataFrame(np.random.randn(6,4), index=dates, columns=['A','B','C','D'])
  8. df.plot()
  9. plt.show()

Pandas - 图2

more for plot()

  1. df.hist(column='A', figsize=(4,3))
  2. df.boxplot(column='A', figsize=(4,3))

Enjoy~

本文由Python腳本GitHub/語雀自動更新

由於語法渲染問題而影響閱讀體驗， 請移步博客閱讀～
本文GitPage地址

GitHub: Karobben
Blog:Karobben
BiliBili:史上最不正經的生物狗