Advanced Python Training at VMWare - Day 2¶
Apr 02-04, 2018 Vikrant Patil
These notes are available online at http://notes.pipal.in/2018/vmware-advanced-apr
© Pipal Academy LLP
Python distribution¶
We will be using anaconda (python 3) distribution for this training. it can be downloaded from
More use cases for decorators¶
In [1]:
%%file trace.py
import os
level = 0
def log(*args):
if os.getenv("DEBUG")=="true":
print(*args)
def trace(f):
def wrapper(*args, **kwargs):
global level
log("| " * level + "|-- " + f.__qualname__, args)
level += 1
value = f(*args)
level -= 1
log("| " * level + "|-- " + "return", value)
return value
return wrapper
In [5]:
%%file fib.py
import sys
from trace import trace
@trace
def fib(n):
if n in [1,2]:
return 1
else:
return fib(n-1) + fib(n-2)
if __name__ == "__main__":
print(fib(int(sys.argv[1])))
In [8]:
!DEBUG="true" python fib.py 12
In [9]:
!time -p python fib.py 30
Recursion gives elegant solution , but creates memory issues, resulting in slow computation. We can make use of tyechnique called memoization to cache the results and avoid repetative calls with same parameters.
In [10]:
%%file memoize.py
def memoize(f):
cache = {}
def wrapper(*args):
if args not in cache:
cache[args] = f(*args)
return cache[args]
return wrapper
In [21]:
%%file fib1.py
import sys
from trace import trace
from memoize import memoize
@memoize
@trace
def fib(n):
if n in [1,2]:
return 1
else:
return fib(n-1) + fib(n-2)
if __name__ == "__main__":
print(fib(int(sys.argv[1])))
In [20]:
!DEBUG="true" python fib1.py 12
In [15]:
!DEBUG="true" python fib.py 12
In [16]:
!time -p python fib1.py 30
In [29]:
def decor1(f):
def wrapper(*args):
print("From decor1")
return f(*args)
return wrapper
def decor2(f):
def wrapper(*args):
print("From decor2")
return f(*args)
return wrapper
@decor1
@decor2
def f():
pass
def g():
pass
In [32]:
def g():
pass
g = decor2(decor1(g))
g()
In [33]:
def g():
pass
g = decor1(decor2(g))
g()
In [34]:
@decor1
@decor2
def g():
pass
g()
In [35]:
%%file functions.py
from collections import deque
def cat(filename):
print(open(filename))
def head(filename, n=5):
with open(filename) as f:
for i in range(n):
print(f.readline(), end="")
def tail(filename, n=5):
history = deque(maxlen=n)
for line in open(filename):
history.append(line.strip())
print("\n".join(history))
In [87]:
%%file functions1.py
from collections import deque
from commands import command, main
@command
def cat(filename):
"""
prints given file to standard output
"""
print(open(filename).read())
@command
def head(filename, n="5"):
with open(filename) as f:
for i in range(int(n)):
print(f.readline(), end="")
@command
def tail(filename, n="5"):
"""
shows last few lines of file
"""
history = deque(maxlen=int(n))
for line in open(filename):
history.append(line.strip())
print("\n".join(history))
if __name__ == "__main__":
main()
In [88]:
%%file commands.py
import sys
cache = {}
def command(f):
global cache
cache[f.__qualname__] = f
return f
def help_():
print("Available commands are")
for name, f in cache.items():
print(name, ":", f.__doc__)
def main():
cmd = sys.argv[1]
args = sys.argv[2:]
if cmd in cache:
f = cache[cmd]
f(*args)
else:
help_()
if __name__ == "__main__":
pass
In [90]:
!python functions1.py cat data.csv
Building data pipelines¶
Advantages of generators
- Lazy evaluation
- Memory efficient
- Due to lazy evaluation we can build pipelines of processing data
In [112]:
import os
def find(root):
for path, dname, filenames in os.walk(root):
for file in filenames:
yield os.sep.join([path, file])
In [105]:
def take(seq, n):
return [next(seq) for i in range(n)]
In [106]:
def integers():
i = 1
while True:
yield i
i += 1
In [107]:
ints = integers()
In [108]:
take(ints, 10)
Out[108]:
In [96]:
take(ints, 10)
Out[96]:
In [98]:
for i in reversed([1,2,3,4,5]):
print(i)
In [99]:
r = reversed([1,2,3])
In [100]:
for i in r:
print(i)
In [101]:
for i in r:
print(i)
In [113]:
files = find(os.getcwd())
In [114]:
take(files, 5)
Out[114]:
In [115]:
[i*i for i in range(10)]
Out[115]:
In [116]:
(i*i for i in range(10))
Out[116]:
In [126]:
def grep(pattern, seq):
import re
p = re.compile(pattern)
return (item for item in seq if p.match(item))
def count(seq):
i = 0
for item in seq:
i +=1
return i
In [143]:
files = find("/home/vikrant/trainings/2018/")
pyfiles = grep("^.*\.py$", files)
take(pyfiles,5)
Out[143]:
In [144]:
def readlines(files):
for file in files:
for line in open(file):
yield line
In [145]:
files = find("/home/vikrant/trainings/2018/")
pyfiles = grep("^.*\.py$", files)
lines = readlines(pyfiles)
funcs = grep("^def .*:$", lines)
count(funcs)
Out[145]:
problems
- is it possible to do vector addition of two infinite sequences? can you write a function for that?
- Write a function get_paragraphs to split given text into paragraphs. The function should take a sequence of lines as argument and returns a sequence of paragraphs. test data
- You are given a sequence of items. Write a function to cunsume duplicate items from sequence while keeping the order. Is it possible to use same function to remove duplicate lines from file.
Iteration protocol¶
In [146]:
def five():
i = 0
while i < 5:
yield i
i += 1
In [147]:
f = five()
In [148]:
f
Out[148]:
In [149]:
next(f)
Out[149]:
In [150]:
next(f)
Out[150]:
In [151]:
next(f)
Out[151]:
In [153]:
next(f)
Out[153]:
In [154]:
next(f)
Out[154]:
In [155]:
next(f)
In [156]:
f = five()
In [157]:
for i in f:
print(i)
In [158]:
next(f)
In [184]:
def get_paragraphs(lines):
para = []
for line in lines:
if line.strip()=="":
if para:
yield "".join(para)
para = []
else:
para.append(line)
if para:
yield "".join(para)
In [183]:
import requests
In [161]:
resp = requests.get("https://ia801405.us.archive.org/12/items/prideandprejudic01342gut/pandp12.txt")
f = open("pnp.txt", "w")
f.write(resp.text)
f.close()
In [162]:
!python functions1.py tail pnp.txt
In [185]:
paras = get_paragraphs(open("pnp.txt"))
In [186]:
count(paras)
Out[186]:
In [171]:
def integers():
i = 1
while True:
yield i
i += 1
In [165]:
n1 = integers()
In [166]:
n2 = integers()
In [167]:
def addseq(seq1, seq2):
for i,j in zip(seq1, seq2):
yield (i+j)
In [168]:
n3 = addseq(n1, n2)
In [169]:
take(n3, 10)
Out[169]:
In [170]:
take(n3, 10)
Out[170]:
In [172]:
n4 = addseq(n1, [1,2,3,4,5])
In [173]:
count(n4)
Out[173]:
In [178]:
def consumedups(seq):
seen = set()
for item in seq:
if item not in seen:
seen.add(item)
yield item
In [179]:
c = consumedups([1,1,2,2,1,4,2,7,2,4,5,2])
In [180]:
for i in c:
print(i, end=",")
In [187]:
%%file bank0.py
balance = 0
def get_balance():
return balance
def withdraw(amount):
global balance
balance -= amount
def deposit(amount):
global balance
balance += amount
if __name__ == "__main__":
pass
In [188]:
%%file bank1.py
def make_account():
return {"balance":0}
def get_balance(account):
return account['balance']
def witdraw(account, amount):
account['balance'] -= amount
def deposit(account, amount):
account["balance"] += amount
if __name__ == "__main__":
a1 = make_account()
a2 = make_account()
a1.get_balance()
In [189]:
class BankAccount:
def __init__(self, balance=0):
self._balance = balance
def get_balance(self):
return self._balance
def deposit(self, amount):
self._balance += amount
def witdraw(self, amount):
self._balance -= amount
In [190]:
a = BankAccount()
In [191]:
BankAccount
Out[191]:
In [192]:
a
Out[192]:
In [193]:
type(a)
Out[193]:
In [195]:
a.__dict__
Out[195]:
In [196]:
a._balance
Out[196]:
In [197]:
a.x = 10
In [198]:
a.__dict__
Out[198]:
In [199]:
a.x
Out[199]:
In [200]:
BankAccount._balance
In [201]:
b = BankAccount()
In [202]:
b.x
In [203]:
a.x
Out[203]:
In [204]:
b.y = 20
In [205]:
b.y
Out[205]:
In [206]:
a.y
In [210]:
class Light:
color = "white"
def __init__(self, status="off"):
self.status = status
In [211]:
l1 = Light()
In [212]:
l2 = Light("on")
In [213]:
l1.status
Out[213]:
In [214]:
l2.status
Out[214]:
In [215]:
l1.color
Out[215]:
In [216]:
l2.color
Out[216]:
In [217]:
Light.color
Out[217]:
In [218]:
Light.color = "yellow"
In [219]:
l1.color
Out[219]:
In [220]:
l2.color
Out[220]:
In [221]:
l1.color = "red"
In [222]:
Light.color
Out[222]:
In [223]:
l2.color
Out[223]:
In [224]:
l1.color
Out[224]:
In [225]:
vars(Light)
Out[225]:
In [226]:
vars(l1)
Out[226]:
In [227]:
vars(l2)
Out[227]:
In [228]:
class Shape:
def area(self):
pass
In [230]:
class Circle(Shape):
def __init__(self, r):
self.radius = r
def area(self):
return 3.14*self.radius**2
class Square(Shape):
def __init__(self, s):
self.side = s
def area(self):
return self.side**2
class Rectangle:
def __init__(self, l, w):
self.len = l
self.width = w
def area(self):
return self.len*self.width
class Poly:
def __init__(self, sides):
self.sides = sides
In [233]:
shapes = [Circle(1), Square(1), Rectangle(1,1)]
In [234]:
areas = [s.area() for s in shapes]
In [235]:
areas
Out[235]:
In [236]:
shapes.extend([Circle(2), Poly([10,10,20,30,12])])
In [237]:
areas = [s.area() for s in shapes]
In [238]:
count(get_paragraphs(open("pnp.txt")))
Out[238]:
In [240]:
count(get_paragraphs(["A\n", "B\n","\n","C"]))
Out[240]:
In [241]:
class Poly1(Shape):
pass
In [242]:
shapes1 = [Circle(1), Square(1), Rectangle(1,1), Poly1()]
In [244]:
[s.area() for s in shapes1]
Out[244]:
In [245]:
class Pair:
def __init__(self, x, y):
self.x = x
self.y = y
In [247]:
p = Pair(1,2)
In [248]:
p
Out[248]:
In [249]:
print(p)
In [250]:
class Pair:
def __init__(self, x, y):
self.x = x
self.y = y
def __str__(self):
return "<{},{}>".format(self.x, self.y)
In [251]:
print(Pair(2,3))
In [253]:
p = Pair(1,2)
In [254]:
p
Out[254]:
In [257]:
class Pair:
def __init__(self, x, y):
self.x = x
self.y = y
def __str__(self):#print uses this, also str
return "<{},{}>".format(str(self.x), str(self.y))
def __repr__(self): # this is used by interpreter, also by repr
return "Pair({},{})".format(repr(self.x), repr(self.y))
In [256]:
Pair(1,2)
Out[256]:
In [258]:
p = Pair(1,2)
In [259]:
str(p)
Out[259]:
In [260]:
repr(p)
Out[260]:
In [262]:
p1 = Pair(0,0)
p2 = Pair(1,1)
In [263]:
p3 = Pair(p1, p2)
In [264]:
print(p3)
In [265]:
p3
Out[265]:
In [275]:
class Pair:
def __init__(self, x, y):
self.x = x
self.y = y
def __str__(self):#print uses this, also str
return "<{},{}>".format(str(self.x), str(self.y))
def __repr__(self): # this is used by interpreter, also by repr
return "Pair({},{})".format(repr(self.x), repr(self.y))
def __add__(self, p):
return Pair(self.x+p.x, self.y+ p.y)
def __sub__(self, p):
return Pair(self.x-p.x, self.y-p.y)
def __mul__(self, c):
return Pair(c*self.x, c*self.y)
def __rmul__(self, c):
return Pair(c*self.x, c*self.y)
def __eq__(self, p):
return self.x==p.x and self.y == p.y
In [276]:
p1 = Pair(1,1)
In [277]:
p1 + p1
Out[277]:
In [278]:
p1 - p2
Out[278]:
In [279]:
p1*2
Out[279]:
In [280]:
2*p1
Out[280]:
In [281]:
p1 == p1
Out[281]:
In [283]:
p1 == Pair(1,1)
Out[283]:
problem
- Implement your own class to represent complex numbers. implement arithmatic operation add/multiplication
- Write a class Timer which can time your programs.
t = Timer() t.start() saxpy() t.stop() print(t.time_taken)
In [288]:
class Foo:
def __init__(self, f):
self.foo = f
def bar(self):
pass
def foobar(self):
self.bar = 0
In [289]:
f = Foo(2)
In [290]:
vars(f)
Out[290]:
In [291]:
f.bar()
In [292]:
f.foobar()
In [293]:
f.bar
Out[293]:
In [294]:
f.bar()
Inheritence¶
In [295]:
class A:
def __init__(self):
print("A.__init__")
class B(A):
def __init__(self):
print("B.__init__")
super().__init__()
class C(B):
def __init__(self):
print("C.__init__")
super().__init__()
In [296]:
A()
Out[296]:
In [297]:
B()
Out[297]:
In [298]:
C()
Out[298]:
In [300]:
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
self.z = (x+y)/2
class ColoredPoint(Point):
def __init__(self, x, y, color):
self.x = x
self.y = y
self.color = color
In [301]:
cp = ColoredPoint(2,3,"red")
In [302]:
cp.z
In [303]:
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
self.z = (x+y)/2
class ColoredPoint(Point):
def __init__(self, x, y, color):
super().__init__(x,y)
self.color = color
In [304]:
cp = ColoredPoint(1,2,"red")
In [305]:
cp.z
Out[305]:
Shape modelling¶
In [312]:
%matplotlib inline
In [356]:
import math
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
def distance(p1, p2):
xdelta = p1.x - p2.x
ydelta = p1.y - p2.y
return math.sqrt(xdelta**2 + ydelta**2)
class Shape:
origin = Point(250,250)
def contains(self, point):
pass
class Circle(Shape):
def __init__(self, r):
self.r = r
def contains(self, p):
return distance(p, self.origin) <= self.r
class Rectangle(Shape):
def __init__(self, w, l):
self.l = l
self.w = w
def contains(self, p):
xd = abs(p.x - self.origin.x)
yd = abs(p.y - self.origin.y)
return xd <= self.l and yd <= self.w
class Union(Shape):
def __init__(self, items):
self.items = items
def contains(self, p):
flag = False
for s in self.items:
flag = flag or s.contains(p)
return flag
class Intersection(Shape):
def __init__(self, items):
self.items = items
def contains(self, p):
flag = True
for s in self.items:
flag = flag and s.contains(p)
return flag
import matplotlib.pyplot as plt
def imshow(img):
plt.imshow(img, cmap=plt.cm.gray)
plt.show()
class Canvas:
def __init__(self, w, l):
self.w = w
self.l = l
self.data = [[False for i in range(w)] for i in range(l)]
def paint(self, shape):
self.data = [[shape.contains(Point(i,j)) or self.data[i][j]
for i in range(self.w)] for j in range(self.l)]
def display(self):
imshow(self.data)
In [344]:
c = Canvas(500, 500)
In [345]:
c1 = Circle(50)
In [346]:
c.paint(c1)
In [347]:
c.display()
In [348]:
r = Rectangle(25,75)
In [349]:
c.paint(r)
In [350]:
c.display()
In [351]:
c = Canvas(500,500)
In [352]:
i = Intersection([Circle(50), Rectangle(30,80)])
In [354]:
c.paint(i)
c.display()
In [357]:
c = Canvas(500, 500)
u = Union([Circle(50), Rectangle(30,80)])
c.paint(u)
c.display()
controlling attributes¶
In [370]:
class Foo:
def __setattr__(self, name, value):
if name not in ["x","y"]:
raise Exception("You can set only x,y")
self.__dict__[name] = value
def __getattr__(self, name):
print("__getattr__", name)
if name in ["x","y"]:
return self.__dict__[name]
else:
raise AttributeError("No such attribute", name)
In [371]:
f = Foo()
In [372]:
f.x = 2
In [373]:
f.x
Out[373]:
In [374]:
f.z = 4
In [375]:
f.z
problem
- Write a immutable class Pair, which can not be modified once created.
In [376]:
In [ ]:
In [377]:
import requests
url = "https://api.github.com/orgs/vmware/repos"
repos = requests.get(url).json()
In [378]:
type(repos)
Out[378]:
In [382]:
for r in repos:
print(r['full_name'], r['forks'])
In [ ]:
In [ ]: