Advanced Python Training at Arcesium - Day 2¶
Oct 22-24, 2018 Vikrant Patil
These notes are available online at http://notes.pipal.in/2018/arcesium-advanced-oct/day2.html
© Pipal Academy LLP
Iterations¶
In [2]:
for c in "some string":
print(c,end=",")
In [4]:
for n in range(10):
print(n, end=",")
In [5]:
def numbers(n):
print("Begin numbers")
i = 0
while i<n:
print("yielding ...")
yield i
print("Back to numbers...")
i += 1
print("Finished numbers!")
In [6]:
five = numbers(5)
In [7]:
five
Out[7]:
In [8]:
next(five)
Out[8]:
In [9]:
next(five)
Out[9]:
In [10]:
next(five)
Out[10]:
In [11]:
next(five)
Out[11]:
In [12]:
next(five)
Out[12]:
In [13]:
next(five)
In [14]:
for i in numbers(4):
print(i, end=",")
In [26]:
def greet():
name = None
while name != "end":
name = yield "Hello, whats your name"
print("Hello ", name)
In [27]:
g = greet()
In [28]:
g.send(None)
Out[28]:
In [29]:
g.send("Vikrant")
Out[29]:
In [30]:
three = numbers(3)
In [31]:
three
Out[31]:
In [32]:
type(three)
Out[32]:
In [33]:
type(g)
Out[33]:
In [34]:
g
Out[34]:
In [35]:
def numbers(n):
print("Begin numbers")
i = 0
while i<n:
print("yielding ...")
yield i
print("Back to numbers...")
i += 1
print("Finished numbers!")
In [36]:
three = numbers(3)
In [37]:
next(three)
Out[37]:
In [38]:
next(three)
Out[38]:
In [39]:
next(three)
Out[39]:
In [40]:
def numbers_sqrs(n):
print("Begin numbers")
i = 0
while i<n:
print("yielding ...")
yield i
print("Back to numbers...1")
yield i*i
print("Back to numbers...2")
i += 1
print("Finished numbers!")
In [43]:
g = numbers_sqrs(3)
In [44]:
next(g)
Out[44]:
In [45]:
next(g)
Out[45]:
In [46]:
next(g)
Out[46]:
In [47]:
next(g)
Out[47]:
In [48]:
next(g)
Out[48]:
In [49]:
next(g)
Out[49]:
In [50]:
next(g)
problem
- Write a generator function
countdownwhich yields numbers top down manner.>>> for n in coutdown(3): ... print(n, end=",") >>> 3,2,1 - Write a function
takewhich makes use of next and list comprehensions to take only first n objects from given generator>>> take(5, countdown(100)) [100,99,98,97,96] - Write a generator which generates infinte series of fibonacci numbers!
In [51]:
def countdown(n):
while n>0:
yield n
n -= 1
In [52]:
def take(seq, n):
return [next(seq) for i in range(n)]
In [56]:
def fib():
prev, cur = 1, 1
yield prev
while True:
prev, cur = cur, prev+cur
yield prev
In [57]:
f = fib()
In [58]:
take(f, 10)
Out[58]:
In [60]:
import os
In [71]:
import os
def find(location=None):
if not location:
location = os.getcwd()
for dirpath, dirnames, files in os.walk(location):
for f in files:
yield os.path.join(dirpath, f)
In [77]:
files = find(location="/home/vikrant/programming/explorations/python/")
In [78]:
take(pyfiles, 5)
Out[78]:
In [74]:
import re
def grep(pattern, seq):
p = re.compile(pattern)
for item in seq:
if p.match(item):
yield item
In [81]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
take(pyfiles, 5)
Out[81]:
In [86]:
def lines(files):
for f in files:
with open(f) as fd:
for line in fd:
yield line
In [87]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
lines_ = lines(pyfiles)
take(grep("^def .*", lines_), 10)
Out[87]:
In [88]:
def count(seq):
return sum(1 for item in seq)
In [89]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
lines_ = lines(pyfiles)
functions = grep("^def .*", lines_)
count(functions)
Out[89]:
In [90]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
count(pyfiles)
Out[90]:
generator expressions¶
In [91]:
squares = (i*i for i in range(100))
In [92]:
squares
Out[92]:
In [93]:
next(squares)
Out[93]:
In [94]:
next(squares)
Out[94]:
In [96]:
def squares(n):
return (i*i for i in range(n))
In [97]:
s100 = squares(100)
In [98]:
next(s100)
Out[98]:
In [99]:
max([i for i in range(10)])
Out[99]:
In [100]:
max(i for i in range(10))
Out[100]:
In [101]:
sum(i*i for i in range(100))
Out[101]:
In [102]:
{a:y for a,y in [(1,1),(2,2)]}
Out[102]:
debuging¶
In [119]:
%%file ssq.py
def SSQ(n):
digits = [int(d) for d in str(n)]
return sum([i*i for i in digits])
def SSQ_(n):
while True:
n = SSQ(n)
yield n
if __name__ == "__main__":
ssq = SSQ_(23)
for i in range(5):
print(next(ssq))
In [104]:
!ls
In [105]:
pdb
In [107]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
dom
count(pyfiles)
In [108]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
dom
count(pyfiles)
In [112]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
dom1
print(count(pyfiles))
In [114]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
dom_
print(count(pyfiles))
In [116]:
del dom1
In [117]:
files = find(location="/home/vikrant/programming/explorations/python/")
pyfiles = grep(".*\.py$", files)
dom
lines_ = lines(pyfiles)
funs = grep("^def.*",lines_)
print(count(funcs))
In [118]:
Out[118]:
profiling¶
In [128]:
%%file saxpy.py
import time
import math
def saxpy(n):
f = 0
for i in range(n):
for j in range(n):
f = f + i*j*1.0
return f
def compute_sqrt(n):
s = 0
for j in range(n):
for i in range(n):
s += math.sqrt(i)
return s
def compute_sqrt1(n):
sqrt = math.sqrt
s = 0
for j in range(n):
for i in range(n):
s += sqrt(i)
return s
def donothing(t=2):
time.sleep(t)
def main():
for i in range(1000, 5000, 1000):
saxpy(i)
compute_sqrt(i)
compute_sqrt1(i)
donothing()
if __name__ == "__main__":
main()
In [129]:
!python -m cProfile saxpy.py
In [127]:
import cProfile
In [132]:
import math
def compute_sqrt(n):
s = 0
for j in range(n):
for i in range(n):
s += math.sqrt(i)
return s
def compute_sqrt1(n):
sqrt = math.sqrt
s = 0
for j in range(n):
for i in range(n):
s += sqrt(i)
return s
In [131]:
cProfile.run("compute_sqrt(1000)")
In [133]:
cProfile.run("compute_sqrt(1000)")
In [134]:
help(cProfile.run)
In [135]:
cProfile.run("compute_sqrt(1000)", filename="sqrt.profiledata")
Numpy¶
In [4]:
import numpy as np
In [2]:
a = np.array([1,2,3,4,5,6,7,8,9])
In [3]:
a
Out[3]:
In [4]:
a.shape
Out[4]:
In [5]:
a.ndim
Out[5]:
In [8]:
a100 = np.arange(100).reshape(10,10)
In [9]:
a100.nbytes
Out[9]:
In [10]:
a100.ndim
Out[10]:
In [11]:
a100.shape
Out[11]:
In [12]:
a100[0]
Out[12]:
In [13]:
a100[0][0]
Out[13]:
In [14]:
a100[0,0]
Out[14]:
In [16]:
a100[:,0]
Out[16]:
In [17]:
a100[0,:]
Out[17]:
In [18]:
a100
Out[18]:
In [20]:
np.zeros(100).reshape(5,20)
Out[20]:
In [21]:
np.zeros_like(a100)
Out[21]:
In [22]:
np.ones_like(a100)
Out[22]:
In [23]:
d = np.asarray([1,2,3,4,5,6,7])
In [24]:
d
Out[24]:
In [26]:
np.empty(10)
Out[26]:
In [27]:
np.empty_like(d)
Out[27]:
In [28]:
np.ones(10, dtype=np.int8)
Out[28]:
In [29]:
d
Out[29]:
In [31]:
subview = a100[:5,:5]
In [32]:
subview
Out[32]:
In [33]:
subview[0,0] = -1
In [34]:
subview
Out[34]:
In [35]:
a100
Out[35]:
In [36]:
scopy = subview.copy()
In [37]:
scopy
Out[37]:
In [38]:
scopy[0,0] = 0
In [39]:
subview
Out[39]:
In [40]:
scopy
Out[40]:
In [41]:
d
Out[41]:
In [42]:
d>3
Out[42]:
In [43]:
d[d>3]
Out[43]:
In [44]:
d - 2
Out[44]:
In [45]:
d2 = d*2
In [46]:
d2
Out[46]:
In [47]:
d
Out[47]:
In [48]:
d + d2
Out[48]:
In [49]:
d * d2
Out[49]:
In [50]:
np.exp(d)
Out[50]:
In [51]:
a100.max()
Out[51]:
In [52]:
a100.mean()
Out[52]:
In [53]:
a100.std()
Out[53]:
In [54]:
a100.cumsum()
Out[54]:
In [57]:
x = np.arange(-5, 5, 0.01)
In [3]:
from scipy.misc import face
In [59]:
image = face(gray=True)
In [60]:
image
Out[60]:
In [2]:
from matplotlib import pyplot as plt
%matplotlib inline
def imshow(img):
plt.imshow(img, cmap=plt.cm.gray)
plt.show()
In [68]:
imshow(image)
In [64]:
negate = 255 - image
In [69]:
imshow(negate)
In [75]:
thumb = image[::3,::3]
In [76]:
thumb.shape
Out[76]:
In [77]:
imshow(thumb)
In [78]:
a100[:,0] = 0
In [79]:
a100
Out[79]:
In [80]:
plain = np.zeros_like(thumb)
In [81]:
imshow(plain)
In [84]:
plain = np.zeros_like(thumb)
plain[::10,:] = 255
plain[:,::10] = 255
In [85]:
imshow(plain)
In [89]:
plain[:11,:11]
Out[89]:
In [90]:
plain = np.zeros(100).reshape(10,10)
plain[::3,:] = 255
plain[:,::3] = 255
imshow(plain)
In [91]:
plain = np.zeros_like(thumb)
plain[::10,:] = 255
plain[:,::10] = 255
In [92]:
imshow(thumb*0.5 + plain*0.5)
In [93]:
imshow(thumb)
In [94]:
imshow(image)
In [95]:
thumb = thumb.copy()
In [97]:
thumb.shape
Out[97]:
In [98]:
def swapcorners(img):
imglike = img.copy()
h, w, = img.shape
q1 = img[:h//2,:w//2].copy()
q4 = img[h//2:,w//2:].copy()
imglike[:h//2,:w//2] = q4
imglike[h//2:,w//2:] = q1
return imglike
In [99]:
imshow(swapcorners(thumb))
In [102]:
thumb = image[::10,::10]
In [103]:
hthumb = np.hstack([thumb,thumb,thumb])
vthump = np.vstack([hthumb, hthumb, hthumb])
imshow(vthump)
Matplotlib¶
Lets look at some data https://notes.pipal.in/2017/arcesium-oct-advpython/HYDERABAD-weather.csv
In [104]:
!wget https://notes.pipal.in/2017/arcesium-oct-advpython/HYDERABAD-weather.csv
In [7]:
import csv
In [8]:
data = csv.reader(open("HYDERABAD-weather.csv"))
In [107]:
import requests
def download(url, filename):
resp = requests.get(url)
with open(filename, "w") as f:
f.write(resp.text)
In [110]:
url = "https://notes.pipal.in/2017/arcesium-oct-advpython/HYDERABAD-weather.csv"
download(url, "wheatherdata.csv")
In [111]:
!tail wheatherdata.csv
In [9]:
data = list(data)
In [10]:
data[0]
Out[10]:
In [11]:
d = data[1:]
In [119]:
def floatcolumn(data, n):
return [float(row[n]) for row in data]
In [121]:
maxtemp = floatcolumn(d, 4)
In [122]:
mintemp = floatcolumn(d, 5)
In [123]:
rainfall = floatcolumn(d, 6)
In [13]:
def float_(sf):
try:
return float(sf)
except Exception as e:
print(e)
return 0
def floatcolumn(data, n):
return [float_(row[n]) for row in data]
In [125]:
rainfall = floatcolumn(d, 6)
In [134]:
plt.scatter(rainfall, maxtemp)
Out[134]:
In [128]:
plt.scatter(rainfall, mintemp)
Out[128]:
In [135]:
data[0]
Out[135]:
In [137]:
year = [int(row[3]) for row in d]
In [138]:
ra = np.array(rainfall)
In [142]:
sorteddata = sorted(d, key=lambda r:r[3])
In [143]:
rainfall = floatcolumn(sorteddata, 6)
In [144]:
year = [int(row[3]) for row in sorteddata]
In [145]:
plt.plot(year, rainfall)
Out[145]:
In [146]:
import random
plt.bar(range(12), [random.random() for i in range(12)])
Out[146]:
In [12]:
months = np.array([row[2] for row in d])
In [15]:
rainfall = np.array(floatcolumn(d, 6))
In [19]:
rainfall[months=="January"].mean()
Out[19]:
In [20]:
def get_mean_rainfall(rainfall, months, month):
return rainfall[months==month].mean()
In [21]:
import datetime
In [22]:
d = datetime.datetime(2000,1,1)
In [23]:
d.strftime("%B")
Out[23]:
In [24]:
mnames = [datetime.datetime(2000,i+1,1).strftime("%B") for i in range(12)]
In [25]:
mnames
Out[25]:
In [26]:
rainfall_ = [get_mean_rainfall(rainfall, months, m) for m in mnames]
In [27]:
rainfall_
Out[27]:
In [31]:
plt.bar(mnames, rainfall_)
In [32]:
plt.bar(range(12), rainfall_)
Out[32]:
In [34]:
plt.pie(rainfall_, labels=mnames, explode=[0]*12)
Out[34]:
pandas¶
In [35]:
from pandas import Series, DataFrame
In [36]:
s = Series(range(5))
In [37]:
s
Out[37]:
In [39]:
s = Series([4,2,7,3],index=['a','b','c','d'])
In [40]:
s
Out[40]:
In [41]:
s['a']
Out[41]:
In [42]:
s[0]
Out[42]:
In [43]:
s['a':'c']
Out[43]:
In [44]:
s = Series({"a":1,"b":2,"c":3,"d":4})
In [45]:
s
Out[45]:
In [48]:
s = Series({"a":1,"b":2,"c":3,"d":4,"j":6}, index=['e','d','c','b','a'])
In [49]:
s
Out[49]:
In [50]:
s[s>2]
Out[50]:
In [51]:
s*s
Out[51]:
In [52]:
s-1
Out[52]:
In [53]:
np.exp(s)
Out[53]:
In [54]:
s.reindex(['a','b','c','d','f','g'])
Out[54]:
In [56]:
s['f'] = 0
In [57]:
s
Out[57]:
In [59]:
months_s = Series(months)
In [62]:
months_s.drop_duplicates()
Out[62]:
In [64]:
d
Out[64]:
In [65]:
data[0]
Out[65]:
In [68]:
df = DataFrame(data[1:], columns=['x', 'city', 'month', 'year', 'maxtemp', 'mintemp', 'rainfall'])
In [69]:
df.describe()
Out[69]:
In [70]:
df.head()
Out[70]:
In [71]:
df.tail()
Out[71]:
In [73]:
import pandas as pd
In [74]:
wd = pd.read_csv("HYDERABAD-weather.csv")
In [75]:
wd.head()
Out[75]:
In [76]:
wd.plot("maxtemp", "rainfall", kind="scatter")
Out[76]:
In [79]:
grpyear = wd.groupby('year').mean()
In [80]:
grpyear.head()
Out[80]:
In [82]:
groupbymonth = wd.groupby("month").mean()
In [84]:
groupbymonth
Out[84]:
In [86]:
del groupbymonth['year']
In [88]:
del groupbymonth['Unnamed: 0']
In [89]:
groupbymonth
Out[89]:
In [91]:
help(groupbymonth.hist)
In [100]:
groupbymonth.plot()
Out[100]:
In [101]:
monthindex = [mnames.index(m) for m in groupbymonth.index]
In [102]:
monthindex
Out[102]:
In [103]:
groupbymonth.index
Out[103]:
In [108]:
groupbymonth['m'] = monthindex
In [109]:
groupbymonth
Out[109]:
In [114]:
groupbymonth.set_index('m').sort_index().plot()
Out[114]:
In [116]:
groupbymonth.reindex(mnames).plot()
Out[116]:
In [117]:
groupbymonth
Out[117]:
In [118]:
groupbymonth.reindex(mnames)
Out[118]:
Why classes ?¶
In [119]:
%%file bank.py
balance = 0
def get_balance():
return balance
def withdraw(amount):
global balance
balance -= amount
def deposit(amount):
global balance
balance += amount
In [120]:
import bank
In [121]:
bank.get_balance()
Out[121]:
In [122]:
bank.deposit(10)
In [124]:
bank.get_balance()
Out[124]:
In [125]:
%%file bank1.py
def make_account():
return {"balance":0}
def get_balance(account):
return account['balance']
def withdraw(account, amount):
account['balance'] -= amount
def deposit(account, amount):
account['balance'] += amount
In [126]:
import bank1
In [127]:
a1 = bank1.make_account()
In [128]:
bank1.get_balance(a1)
Out[128]:
In [130]:
bank1.get_balance(a1)
Out[130]:
modules¶
In [131]:
%%file mymodule.py
x = 0
print("location 1")
if __name__ == "__main__":
print("location 2")
In [132]:
import mymodule
In [133]:
!python mymodule.py
In [134]:
%%file mymodule1.py
x = 0
print(__name__)
In [135]:
import mymodule1
In [136]:
!python mymodule1.py
In [137]:
%%file backup.py
def backup():
# go to some location
# delete few old unrequired files
# copy new uopdated files from specfied location
pass
backup()
In [138]:
!python backup.py
In [139]:
%%file backup.py
def backup():
# go to some location
# delete few old unrequired files
# copy new uopdated files from specfied location
pass
if __name__ == "__main__":
backup()
In [140]:
!python backup.py
In [141]:
import math
In [142]:
math.pi
Out[142]:
In [ ]: