Sep 27-29, 2017 Vikrant Patil
These notes are available online at http://notes.pipal.in/2017/vmware-pune-advpy
© Pipal Academy LLP
Problem: Write a module cmdline.py to build command-line application easily.
%%file commands.py
from cmdline import command, main
@command
def hello():
"""
print hello
"""
print("Hello world!")
@command
def cat(filename):
"""
prints given file on standard output
"""
for line in open(filename).readlines():
print(line.strip())
@command
def grep(word, filename):
"""
greps given word in given file
"""
for line in open(filename).readlines():
if word in line:
print(line.strip())
#grep("hello", "hello.py")
if __name__ == "__main__":
main()
This is how it should work
$ python commands.py hello
Hello World!
$ python commands.py cat commands.py
from cmdline import command, main
@command
def hello():
"""
print hello
"""
print("Hello world!")
@command
def cat(filename):
"""
prints given file on standard output
"""
for line in open(filename).readlines():
print(line.strip())
@command
def grep(word, filename):
"""
greps given word in given file
"""
for line in open(filename).readlines():
if word in line:
print(line.strip())
if __name__ == "__main__":
main()%%file cmdline.py
import sys
commands = {}
def command(f):
commands[f.__name__] = f
return f
def help():
print("Available command:")
for cmd in commands.keys():
print(cmd.ljust(10), commands[cmd].__doc__.strip())
def main():
cmdname = sys.argv[1]
args = sys.argv[2:]
if "help" in sys.argv:
help()
elif cmdname in commands:
func = commands[cmdname]
print("Executing command:", cmdname)
func(*args)
!python commands.py hello
!python commands.py cat commands.py
!python commands.py grep hello commands.py
!python commands.py help
@with_retries(retries=3, delay=0.1)
wget(url
.
..
@debug(prefix="[INFO]: ")
def fib(n):
@login_required(role=admin)
def edit_interface(..):
.
.import time
def with_retries(retries=5, delay=0):
def decor(f):
def g(*args):
print("retries=", retries, "delay=",delay)
for i in range(retries):
try:
return f(*args)
except Exception as e:
print(f.__name__, args, "failed", e)
time.sleep(delay)
print("Giving up!")
return g
return decor
from urllib.request import urlopen
@with_retries(retries=3, delay=0.5)
def wget(url):
return urlopen(url).read()
def wget1(url):
return urlopen(url).read()
wget("http://helloksdkjdshfkjhds.jhfjhd.com")
decor = with_retries(retries=3, delay=0.4)
f = decor(wget1)
f("http://kjshadkjh.skjadh")
from functools import partial
def with_retries(f=None, retries=5, delay=0):
if f == None:
return partial(with_retries, retries=retries, delay=delay)
def g(*args):
print("retries=", retries, "delay=",delay)
for i in range(retries):
try:
return f(*args)
except Exception as e:
print(f.__name__, args, "failed", e)
time.sleep(delay)
print("Giving up!")
return g
def square(x):
"""
compute square of a number
"""
return x*x
help(square)
!cat fib.py
from fib import fib
help(fib)
print(fib)
from functools import wraps
def debug(func):
@wraps(func)
def wrapper(*args):
print("Debug: === ", func.__qualname__, args)
return func(*args)
return wrapper
@debug
def add(a, b):
"""
adds a and b
"""
return a+b
add(1,2)
help(add)
print(add)
just observe the for loops in python
for n in [1,2, 3, 4, 5]:
print(n)
for c in "string":
print(c)
for key in {'x':1, 'y':2}:
print(key)
%%file nums.txt
one
two
three
for line in open("nums.txt"):
print(repr(line))
items = [1, 2, 3]
itr = iter(items)
next(itr)
next(itr)
next(itr)
next(itr)
def square(numbers):
for n in numbers:
yield n*n
sq5 = square(range(1,6))
sq5
for s in sq5:
print(s)
def squares(numbers):
print("Begin squares")
for n in numbers:
print("computing square of", n)
yield n*n
print("Finish squares")
sq4 = squares(range(1, 5))
sq4
next(sq4)
next(sq4)
next(sq4)
next(sq4)
next(sq4)
for x in squares([1,2,3,4]):
print(x)
a = squares(range(4))
b = a
next(a)
next(b)
x = squares(range(3))
y = squares(range(3))
next(x)
next(y)
problem: Write a generator countdown that takes number n as argument and generates all numbers down to 0.
>>> for i in countdown(3):
print(i)
3
2
1
0 problem: Write a generator triangular that takes number n as argument and generates sequence of first n triangular numbers. nth triangular number is sum of first n natural numbers.
>>> for t in triangular(5):
print(t)
1
3
6
10
15[n*n for n in range(5)]
(n*n for n in range(10))
sq = (n*n for n in range(10))
for s in sq:
print(s)
sum((x*x for x in range(1000000)))
sum(x*x for x in range(1000000))
sum(list([x*x for x in range(100000)]))
def integers():
"""
lets generate infinite sequence of natural numbers
"""
i = 1
while True:
yield i
i += 1
def squares(numbers):
for n in numbers:
yield n*n
#return (n*n for n in numbers)
def take(n , seq):
it = iter(seq)
return list(next(it) for i in range(n))
take(10, squares(integers()))
import os
def find(root):
for path, dirname, filenames in os.walk(root):
for f in filenames:
yield os.path.join(path, f)
def grep(pattern, seq):
return (x for x in seq if pattern in x)
files = find(".")
pyfiles = grep(".py", files)
print(take(10, pyfiles))
def count(seq):
i = 0
for x in seq:
i += 1
return i
count([3,4,5,6,5])
def count(seq):
return sum(1 for x in seq)
count(range(100))
files = find(".")
pyfiles = grep(".py", files)
print(count(pyfiles))
def readlines(filenames):
"""
Returns an iterators over lines in all files specified
"""
for f in filenames:
for line in open(f):
yield line
How many lines of python code we have written in this course?
files = find(".")
pyfiles = grep(".py", files)
lines = readlines(pyfiles)
print(count(lines))
How many function definations we have?
files = find(".")
pyfiles = grep(".py", files)
lines = readlines(pyfiles)
functions = grep("def", lines)
print(count(functions))
problem: Write a function get_paragraphs to split given text into paragraphs. The function should take a sequence of lines as argument and returns a sequence of paragraphs. Assume that there is an empty line after every paragraph
For sample input, see: (http://anandology.com/tmp/pg1342.txt) once the function is there , we should be able to find:
files = find("/tmp/")
txtfiles = grep(".txt", files) # there is a huge file data.txt of size 5.4GB
lines = readlines(txtfiles) # get stream of lines from that file
hellos = grep("hello", lines) # grep for hello in those lines
print(count(hellos))
def get_paragraphs(lines):
paragraph = []
for line in lines:
if line.strip() != "":
paragraph.append(line)
elif paragraph:
yield "".join(paragraph)
paragraph = []
if paragraph:
yield "".join(paragraph)
lines = ["A1\n", "A2\n", "\n", "B1\n", "\n", "C1\n", "C2\n","C3\n"]
p = get_paragraphs(lines=lines)
next(p)
next(p)
next(p)
p1 = get_paragraphs(open("/tmp/pg1342.txt"))
max(p1, key=len)
next(p1)
p2 = get_paragraphs(open("/tmp/pg1342.txt"))
count(p2)
from urllib.request import urlopen
response = urlopen("http://httpbin.org/html")
response
contents = response.read()
contents[:100]
html = contents.decode("utf-8")
print(html[:400])
response.status
There is a third party library requests which makes handling http requests very handy
install it using
pip3 install reuestimport requests
response = requests.get("http://httpbin.org/html")
print(response.text[:400])
response.headers
response.status_code
response = requests.get("http://httpbin.org/get", params={"query":"python class", "page":"2"})
print(response.text)
response = requests.post("http://httpbin.org/post", data="Plain text data")
print(response.text)
response = requests.post("http://httpbin.org/post", data={"name":"Python", "date":"28th sep 2017"})
print(response.text)
import requests
url = "https://api.github.com/orgs/vmware/repos"
repos = requests.get(url).json()
type(repos)
for repo in repos:
print(repo["full_name"], repo['forks'])
def get_forks(repo):
return repo['forks']
repos = sorted(repos, key=get_forks, reverse=True)[:5]
for r in repos:
print(r['full_name'], r['forks'])
def get_top_contributors(reponame):
url = "https://api.github.com/repos/{}/stats/contributors".format(reponame)
print(url)
contributors = requests.get(url).json()
contributors = sorted(contributors, key=lambda x:x['total'], reverse=True)
for c in contributors[:5]:
print(c['author']['login'], c['total'])
get_top_contributors("vmware/pyvmomi")
How to pass username and password?
username = "xxx"
passwd = open("somesecretfile").read().strip()
resp = requests.get(url, auth=(username, passwd))Problem: How do you find distance between two cities using google
hints:
- url for map api is (https://maps.googleapis.com/maps/api/distancematrix/json)
- parameters for above get request are
origins
destinations
units (metric)import requests
def distance(origin, destination):
url = "https://maps.googleapis.com/maps/api/distancematrix/json"
response = requests.get(url, params={"units":"metric",
"origins":origin,
"destinations":destination
})
data = response.json()
return data['rows'][0]['elements'][0]['distance']['text']
distance("pune", "dapoli")
%%file t1.py
import threading
def task():
print("Hello ", threading.currentThread().getName())
def main():
t1 = threading.Thread(target=task)
t1.start()
t1.join()
if __name__ == "__main__":
main()
!python t1.py
Lets run it with 10 threads
%%file t2.py
import threading
def task():
print("Hello ", threading.currentThread().getName())
def main():
threads = [threading.Thread(target=task) for i in range(10)]
for t in threads:
t.start()
for t in threads:
t.join()
if __name__ == "__main__":
main()
!python t2.py
%%file counter.py
import threading
class Counter:
def __init__(self):
self.count = 0
def tick(self):
self.count += 1
def task(counter, n):
for i in range(n):
counter.tick()
def main():
counter = Counter()
n = 100000
nthreads = 10
threads = [threading.Thread(target=task, args=(counter, n)) for i in range(nthreads)]
for t in threads:
t.start()
for t in threads:
t.join()
print(counter.count)
if __name__ == "__main__":
main()
!python counter.py
!python counter.py
!time -p python counter.py
%%file counter2.py
import threading
class Counter:
def __init__(self):
self.count = 0
self.lock = threading.Lock()
def tick(self):
with self.lock:
self.count += 1
def task(counter, n):
for i in range(n):
counter.tick()
def main():
counter = Counter()
n = 100000
nthreads = 10
threads = [threading.Thread(target=task, args=(counter, n)) for i in range(nthreads)]
for t in threads:
t.start()
for t in threads:
t.join()
print(counter.count)
if __name__ == "__main__":
main()
!python counter2.py
!time -p python counter2.py
!time -p python counter.py
!seq 10 | xargs printf "http://httpbin.org/get?x=%d\n"
!seq 10 | xargs printf "http://httpbin.org/get?x=%d\n" >urls.txt
%%file sget.py
import sys
from urllib.request import urlopen
def get_urls(filename):
return [line.strip() for line in open(filename)]
def wget(url):
return urlopen(url).read()
def main():
filename = sys.argv[1]
for url in get_urls(filename):
wget(url)
if __name__ == "__main__":
main()
!time -p python sget.py urls.txt
%%file pget.py
import sys
from sget import get_urls, wget
from multiprocessing.pool import ThreadPool
def main():
filename = sys.argv[1]
concurrency = int(sys.argv[2])
urls = get_urls(filename)
pool = ThreadPool(concurrency)
pool.map(wget, urls)
if __name__ == "__main__":
main()
!time -p python pget.py urls.txt 1
!time -p python pget.py urls.txt 2
!time -p python pget.py urls.txt 4
!time -p python pget.py urls.txt 8
import sget
get_urls
sget.get_urls
sget.wget
from sget import get_urls
get_urls
%%file pcpu.py
import sys
from multiprocessing.pool import ThreadPool
def task(dummy):
sum = 0
for i in range(1000):
for j in range(10000):
sum += 1.0*i*j
return sum
def main():
concurrency = int(sys.argv[1])
pool = ThreadPool(concurrency)
pool.map(task, range(10))
if __name__ == "__main__":
main()
!time -p python pcpu.py 1
!time -p python pcpu.py 2
!time -p python pcpu.py 4
%%file pcpu1.py
import sys
from multiprocessing.pool import Pool
def task2(dummy):
sum = 0
for i in range(1000):
for j in range(10000):
sum += 1.0*i*j
return sum
def task(dummy):
sum = 0
for i in range(1000):
for j in range(10000):
sum += 1.0*i*j
return sum
def main():
concurrency = int(sys.argv[1])
pool = Pool(concurrency)
pool.map(task, range(10))
pool.map(task2, range(10))
if __name__ == "__main__":
main()
!time -p python pcpu1.py 1
!time -p python pcpu1.py 2
!time -p python pcpu1.py 4
Refernce: google for Concurrency from Ground up - video by David Beazley
"Wizard of oz is {}".format("python")
"wizard of {} is in {}".format("python", "oz")
"purpose of life is {0}, my compuation in {1} says so!".format(42, "python")
for i in range(1, 11):
line = "{0:2d} {1:3d} {2:4d}".format(i, i*i, i*i*i)
print(line)
for i in range(1, 11):
line = "{ints:2d} {sqrs:3d} {cubes:4d}".format(ints=i, sqrs=i*i, cubes=i*i*i)
print(line)
"12".zfill(10)
"12.25".zfill(10)
"center".center(50)
"center".rjust(50)
"center".ljust(50)
>>> pascal(3)
[[1],[1,1],[1,2,1]] >>> print_pascal(5)
1
1 1
1 2 1
1 3 3 1
1 4 6 4 1