Practical Python Master Class at VMWare - Day 3

Sep 26-28, 2018 Vikrant Patil

These notes are available online at http://notes.pipal.in/2018/vmware-practical-master-sep/

© Pipal Academy LLP

Day 1 | Day 2 | Day 3

Python distribution

We will be using anaconda (python 3) distribution for this training. it can be downloaded from

https://www.anaconda.com/download/

commandline tools

%%file wc.py
import sys

def linecount(filename):
    with open(filename) as f:
        return len(f.readlines())
    
def wordcount(filename):
    with open(filename) as f:
        return len(f.read().split())
    
def charcount(filename):
    with open(filename) as f:
        return len(f.read())



if __name__ == "__main__":
    file = sys.argv[1]
    print(linecount(file), wordcount(file), charcount(file), file)

!python wc.py day1.html

sys.argv is simplest way of taking arguments from commandline. But you will have to mange your arguments on your own. it will be very difficult to manage script with complicated arguments.

We will use argparse module to do this for us.

%%file grep.py
import argparse


def parse_args():
    parser = argparse.ArgumentParser("grep")
    ### this is the way to add positional arguments
    parser.add_argument("pattern", help="Pattern to look for")
    parser.add_argument("filename", help="Look for pattern in this file")
    
    return parser.parse_args()

def grep(pattern, file):
    with open(file) as f:
        for line in f:
            if pattern in line:
                print(line,end="")

if __name__ == "__main__":
    args = parse_args()
    print(args)
    grep(args.pattern, args.filename)

!python grep.py --help

!python grep.py "def" grep.py

%%file grep1.py
import argparse


def parse_args():
    parser = argparse.ArgumentParser("grep")
    ### this is the way to add positional arguments
    parser.add_argument("pattern", help="Pattern to look for")
    parser.add_argument("filename", help="Look for pattern in this file")
    #this is how you add optional argument
    
    parser.add_argument("-m", "--maxcount", 
                        type=int,
                        help="How many lines to match. command will stop after printing maxcount matches")
    
    return parser.parse_args()


def grepmax(pattern, file, maxcount):
    lines = grep(pattern, file)
    return [next(lines) for i in range(maxcount)]
    

def grep(pattern, file):
    with open(file) as f:
        for line in f:
            if pattern in line:
                yield line
                
def printlines(lines):
    for line in lines:
        print(line, end="")
                
                
if __name__ == "__main__":
    args = parse_args()
    print(args)
    if args.maxcount:
        printlines(grepmax(args.pattern, args.filename, args.maxcount))
    else:
        printlines(grep(args.pattern, args.filename))

!python grep1.py --help

!python grep1.py "div" day1.html

!python grep1.py  -m 5 "div" day1.html

!python grep1.py  --maxcount 5 "div" day1.html

from collections import deque

dq = deque(maxlen=5)

dq.append(1)

dq.append(2)

dq.append(3)

dq.append(4)

dq.append(5)

dq

dq.append(6)

dq

%%file grep2.py
import argparse
from collections import deque

def parse_args():
    parser = argparse.ArgumentParser("grep")
    ### this is the way to add positional arguments
    parser.add_argument("pattern", help="Pattern to look for")
    parser.add_argument("filename", help="Look for pattern in this file")
    #this is how you add optional argument
    
    parser.add_argument("-m", "--maxcount", 
                        type=int,
                        help="How many lines to match. command will stop after printing maxcount matches")
    parser.add_argument("-c", "--context",
                        type=int, default=1,
                        help="Show context of these many lines for match") 
    
    #this how you add flag, when flag is present it is taken as true
    parser.add_argument("-v", "--invert",action="store_true", 
                        help="Invert the match")
    
    return parser.parse_args()

def take(seq, n):
    return [next(seq) for i in range(n)]

def grep(pattern, file, context):
    q = deque(maxlen=context)
    with open(file) as f:
        for line in f:
            q.append(line)
            if pattern in line:
                for l in q:
                    yield l
                q.clear()
                
def printlines(lines):
    for line in lines:
        print(line, end="")
                
                
if __name__ == "__main__":
    args = parse_args()
    print(args)
    if args.maxcount:
        greped = grep(args.pattern, args.filename, args.context)
        printlines(take(greped, args.maxcount*args.context))
    else:
        printlines(grep(args.pattern, args.filename, args.context))

def f(n):
    yield from range(n)

list(f(4))

!python grep2.py --help

!python grep2.py -m2 -c3 "def" grep2.py 

mutually excusive options

parser = argparse.ArgumentParser(prog='PROG')
group = parser.add_mutually_exclusive_group()
group.add_argument('--foo', action='store_true')
group.add_argument('--bar', action='store_false')
parser.parse_args(['--foo'])

parser.parse_args(["--foo", "--bar"])

parser.parse_args([ "--bar"])

Explore more on adding subparser, subcommands

Using click to build CLI tools quickly

%%file hello.py
import click


@click.command()
@click.argument("name")
def hello(name):
    print("hello", name, "!")
    
if __name__ == "__main__":
    hello()
    

    

!python hello.py

!python hello.py --help

%%file hello.py
import click


@click.command()
@click.argument("name")
@click.argument("greeting")
def hello(name, greeting):
    print("hello", name, greeting, "!")
    
if __name__ == "__main__":
    hello()
    

!python hello.py --help

%%file hello.py
#!/home/vikrant/usr/local/anaconda3/bin/python3
import click


@click.command()
@click.option("--upper", is_flag=True, help="Print in upper case")
@click.argument("name")
@click.argument("greeting")
def hello(name, greeting, upper):
    s =  " ".join(["hello", name, greeting, "!"])
    if upper:
        print(s.upper())
    else:
        print(s)

if __name__ == "__main__":
    hello()
    

!python hello.py --upper vmware morning

%%file hello.py
import click
import sys


@click.command()
@click.option("--upper", is_flag=True, help="Print in upper case")
@click.argument("name")
@click.argument("greeting")
def hello(name, greeting, upper):
    s =  " ".join(["hello", name, greeting, "!"])
    if upper:
        print(s.upper())
    else:
        print(s)


@click.command()
@click.argument("name1")
def echo(name1):
    print("echo", name1)
        
if __name__ == "__main__":
    if sys.argv[1]=="hello":
        hello()
    elif sys.argv[1]=="echo":
        echo()
    elif sys.argv[1]=="--help":
        print("subcomands are:\nhello\necho")

!python hello.py --help

!python hello.py echo --help

!python hello.py hello --help

!python hello.py hello vmware morning

Packaging python projects

import os

Directory as module!

os.mkdir("package")

os.makedirs("package/foo/nested")

!touch package/foo/__init__.py

!touch package/foo/nested/__init__.py

!tree package/

os.getcwd()

os.chdir("package/")

os.getcwd()

import foo

from foo import nested

!cat >> foo/hello.py
print("hello.py")

!cat foo/hello.py

from foo import hello

hello.add(2,3)

%%file setup.py
from distutils.core import setup

setup(
    name="foo",
    version='1.0',
    description="Fool library for packaging demo!",
    author="Vikrant",
    email="vikrant@foo.net",
    url="www.foo.net",
    packages=["foo", "foo.nested"],
)

!tree

os.mkdir("bar")

!python setup.py sdist #source distribution

!tree

os.chdir("dist")

!ls

import shutil

shutil.unpack_archive("foo-1.0.tar.gz")

!lsal

!python3.5 -m venv fooenv

!ls

$ source fooenv/bin/activate
(fooenv)$ cd foo-1.0
(fooenv)$ python setup.py install

this will install your foo package in "fooenv" virtual environment. then you can start python interpreter and can import foo module

!pwd

os.chdir("/home/vikrant/trainings/2018/vmware-practical-master-sep")

os.mkdir("executable_folder")

os.chdir("executable_folder/")

%%file __main__.py

print("Hello from executable folder")

%%file __init__.py
pass

os.chdir("/home/vikrant/trainings/2018/vmware-practical-master-sep")

!python executable_folder/

shutil.make_archive(format="zip", base_name="executable_folder", base_dir="executable_folder/")

!python executable_folder.zip

Handling dependencies using requirements.txt

pip freeze >requirements.txt will collect all the dependencies's names in requirements.txt

pip install -r requirements.txt this command installs all dependecies listed in requirements.txt

Invoking External Commands

import subprocess

p = subprocess.run(["ls","-l"])

p.args

results.stdout

results.stderr

r = subprocess.run(["ls", "-l"], stdout=subprocess.PIPE) 

r.stdout

r.stdout.decode("utf-8")

r = subprocess.run(["cat", "xyz.txt"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

r.stdout

r.stderr

subprocess.check_output(["ls","-l"])

subprocess.run(["ls", "-l"], stdout=subprocess.PIPE).stdout

Writing to stderr and stdout

import sys

sys.stdout.write("hello world")

print("hello")

sys.stderr.write("Error!")

Communicating with process

%%file echo.py

import sys



if __name__ == "__main__":

    for line in sys.stdin:
        sys.stderr.write("Debug: got line"  + line)
        sys.stdout.write(line)
    

p = subprocess.Popen(["python", "echo.py"],
                       stdin=subprocess.PIPE,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE)

p

lines = b"""one
two
three
four
five
"""

stdout, stderr = p.communicate(lines)

print(stdout.decode("utf-8"))

print(stderr.decode("utf-8"))

problem

• modify our original wc.py program to read data from stdin, so that it can work in unix pipe!

Multithreading

import threading

import time
def tick(n):
    for i in range(n):
        time.sleep(2)
        print("Tick-", i)

def rtick(n):
    for i in reversed(range(n)):
        time.sleep(1)
        print("Reversed Tick-", i)

t = threading.Thread(target=tick, args=(10,))

t.start()

t = threading.Thread(target=tick, args=(10,))

t1 = threading.Thread(target=rtick, args=(10,))

t.start()

t1.start()

t.join()
print("Control back to main thread!")

(1,2)

(1)

(1,)

%%file wthreads.py
import sys
import threading
import requests

def download(url):
    r = requests.get(url)
    
def tasks(concurrency):
    
    urls = ["http://httpbin.org/get" for i in range(10)]
    
    while urls:
        tcount = concurrency if len(urls) >= concurrency else len(urls)
        print("Starting {} threads".format(tcount))
        threads = [threading.Thread(target=download, args=(urls.pop(),) )for i in range(tcount)]

        for t in threads:
            t.start()
        for t in threads:
            t.join()
            
if __name__ == "__main__":
    tasks(int(sys.argv[1]))
    
    

!time -p python wthreads.py 1

!time -p python wthreads.py 2

l.pop()

%%file cthreads.py
import sys
import threading
import requests

def saxpy(url):
    s = 0.0
    for i in range(10000):
        for j in range(1000):
            s += i*j*1.0
    
def tasks(concurrency):
    
    urls = ["http://httpbin.org/get" for i in range(10)]
    
    while urls:
        tcount = concurrency if len(urls) >= concurrency else len(urls)
        print("Starting {} threads".format(tcount))
        threads = [threading.Thread(target=saxpy, args=(urls.pop(),) )for i in range(tcount)]

        for t in threads:
            t.start()
        for t in threads:
            t.join()
            
if __name__ == "__main__":
    tasks(int(sys.argv[1]))
    
    

!time -p python cthreads.py 2

!time -p python cthreads.py 1

%%file tpool.py
import sys
import requests
from multiprocessing.pool import ThreadPool

def task(url):
    r = requests.get(url)
    
    
def threadpool(concurrency):
    pool = ThreadPool(concurrency)
    urls = ["http://httpbin.org/get" for i in range(10)]
    r = pool.map(task, urls)
    
if __name__ == "__main__":
    threadpool(int(sys.argv[1]))

!time -p python tpool.py 1

!time -p python tpool.py 2

!time -p python tpool.py 4

%%file processpool.py
import sys
import requests
from multiprocessing.pool import Pool


def saxpy(url):
    s = 0.0
    for i in range(10000):
        for j in range(1000):
            s += i*j*1.0
    return s
    
def processpool(concurrency):
    pool = Pool(concurrency)
    urls = ["http://httpbin.org/get" for i in range(10)]
    #r = pool.map(saxpy, urls)
    #r = pool.imap(saxpy, urls) #
    #print(next(r))
    r = pool.apply_async(saxpy, (urls.pop,))
    print(r.get(timeout=5))
    
if __name__ == "__main__":
    processpool(int(sys.argv[1]))

!time -p python processpool.py 1

!time -p python processpool.py 2

How to communicate between threads

import threading
import time

class Clock(threading.Thread):
    def __init__(self , *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.event = threading.Event()
        self.count = 0
        self.lock = threading.Lock()
        
    def run(self):
        while not self.event.isSet():
            print("Tick-", self.count)
            self.lock.acquire()
            self.count += 1
            self.lock.release()
            time.sleep(2)
            
    def stop(self):
        self.event.set()
        
        
    def reset(self):
        self.lock.acquire()
        self.count = 0
        self.lock.release()

c = Clock()

c.start()

Tick- 0
Tick- 1
Tick- 2
Tick- 3

c.reset()

Tick- 0
Tick- 1
Tick- 2
Tick- 3
Tick- 4
Tick- 5
Tick- 6
Tick- 7

c.stop()

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-7-77b2c7ac3c9b> in <module>()
----> 1 l.acquire()

KeyboardInterrupt: 

Working with rest api

import requests

r = requests.get("http://httpbin.org/get")

print(r.text)

{
  "args": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.14.2"
  }, 
  "origin": "42.106.206.76", 
  "url": "http://httpbin.org/get"
}

r.status_code

200

r = requests.get("http://httpbin.org/html")

r.status_code

200

print(r.text[:100])

<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
      <h1>Herman Melville - Moby-Dick</h1>

     

r = requests.get("http://httpbin.org/get", params={"query":"xyz","abc":"hello","language":"python"})

print(r.text)

{
  "args": {
    "abc": "hello", 
    "language": "python", 
    "query": "xyz"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.14.2"
  }, 
  "origin": "42.106.236.172", 
  "url": "http://httpbin.org/get?query=xyz&abc=hello&language=python"
}

rr = requests.post("http://httpbin.org/post", data="plain text")

print(rr.text)

{
  "args": {}, 
  "data": "plain text", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "10", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.14.2"
  }, 
  "json": null, 
  "origin": "42.106.206.76", 
  "url": "http://httpbin.org/post"
}

r = requests.post("http://httpbin.org/post", data={"name":"python","email":"py@xyz.com"})

print(r.text)

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "email": "py@xyz.com", 
    "name": "python"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "30", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.14.2"
  }, 
  "json": null, 
  "origin": "42.106.236.172", 
  "url": "http://httpbin.org/post"
}

url = "https://api.github.com/orgs/vmware/repos"
repos = requests.get("https://api.github.com/orgs/vmware/repos").json()

type(repos)

list

repos[0]

{'archive_url': 'https://api.github.com/repos/vmware/pyvco/{archive_format}{/ref}',
 'archived': False,
 'assignees_url': 'https://api.github.com/repos/vmware/pyvco/assignees{/user}',
 'blobs_url': 'https://api.github.com/repos/vmware/pyvco/git/blobs{/sha}',
 'branches_url': 'https://api.github.com/repos/vmware/pyvco/branches{/branch}',
 'clone_url': 'https://github.com/vmware/pyvco.git',
 'collaborators_url': 'https://api.github.com/repos/vmware/pyvco/collaborators{/collaborator}',
 'comments_url': 'https://api.github.com/repos/vmware/pyvco/comments{/number}',
 'commits_url': 'https://api.github.com/repos/vmware/pyvco/commits{/sha}',
 'compare_url': 'https://api.github.com/repos/vmware/pyvco/compare/{base}...{head}',
 'contents_url': 'https://api.github.com/repos/vmware/pyvco/contents/{+path}',
 'contributors_url': 'https://api.github.com/repos/vmware/pyvco/contributors',
 'created_at': '2011-03-23T18:18:26Z',
 'default_branch': 'master',
 'deployments_url': 'https://api.github.com/repos/vmware/pyvco/deployments',
 'description': 'Python bindings for VMware Orchestrator',
 'downloads_url': 'https://api.github.com/repos/vmware/pyvco/downloads',
 'events_url': 'https://api.github.com/repos/vmware/pyvco/events',
 'fork': True,
 'forks': 3,
 'forks_count': 3,
 'forks_url': 'https://api.github.com/repos/vmware/pyvco/forks',
 'full_name': 'vmware/pyvco',
 'git_commits_url': 'https://api.github.com/repos/vmware/pyvco/git/commits{/sha}',
 'git_refs_url': 'https://api.github.com/repos/vmware/pyvco/git/refs{/sha}',
 'git_tags_url': 'https://api.github.com/repos/vmware/pyvco/git/tags{/sha}',
 'git_url': 'git://github.com/vmware/pyvco.git',
 'has_downloads': True,
 'has_issues': False,
 'has_pages': True,
 'has_projects': True,
 'has_wiki': True,
 'homepage': 'http://sigma.github.com/vmw.vco',
 'hooks_url': 'https://api.github.com/repos/vmware/pyvco/hooks',
 'html_url': 'https://github.com/vmware/pyvco',
 'id': 1517508,
 'issue_comment_url': 'https://api.github.com/repos/vmware/pyvco/issues/comments{/number}',
 'issue_events_url': 'https://api.github.com/repos/vmware/pyvco/issues/events{/number}',
 'issues_url': 'https://api.github.com/repos/vmware/pyvco/issues{/number}',
 'keys_url': 'https://api.github.com/repos/vmware/pyvco/keys{/key_id}',
 'labels_url': 'https://api.github.com/repos/vmware/pyvco/labels{/name}',
 'language': 'Python',
 'languages_url': 'https://api.github.com/repos/vmware/pyvco/languages',
 'license': {'key': 'mit',
  'name': 'MIT License',
  'node_id': 'MDc6TGljZW5zZTEz',
  'spdx_id': 'MIT',
  'url': 'https://api.github.com/licenses/mit'},
 'merges_url': 'https://api.github.com/repos/vmware/pyvco/merges',
 'milestones_url': 'https://api.github.com/repos/vmware/pyvco/milestones{/number}',
 'mirror_url': None,
 'name': 'pyvco',
 'node_id': 'MDEwOlJlcG9zaXRvcnkxNTE3NTA4',
 'notifications_url': 'https://api.github.com/repos/vmware/pyvco/notifications{?since,all,participating}',
 'open_issues': 0,
 'open_issues_count': 0,
 'owner': {'avatar_url': 'https://avatars0.githubusercontent.com/u/473334?v=4',
  'events_url': 'https://api.github.com/users/vmware/events{/privacy}',
  'followers_url': 'https://api.github.com/users/vmware/followers',
  'following_url': 'https://api.github.com/users/vmware/following{/other_user}',
  'gists_url': 'https://api.github.com/users/vmware/gists{/gist_id}',
  'gravatar_id': '',
  'html_url': 'https://github.com/vmware',
  'id': 473334,
  'login': 'vmware',
  'node_id': 'MDEyOk9yZ2FuaXphdGlvbjQ3MzMzNA==',
  'organizations_url': 'https://api.github.com/users/vmware/orgs',
  'received_events_url': 'https://api.github.com/users/vmware/received_events',
  'repos_url': 'https://api.github.com/users/vmware/repos',
  'site_admin': False,
  'starred_url': 'https://api.github.com/users/vmware/starred{/owner}{/repo}',
  'subscriptions_url': 'https://api.github.com/users/vmware/subscriptions',
  'type': 'Organization',
  'url': 'https://api.github.com/users/vmware'},
 'permissions': {'admin': False, 'pull': True, 'push': False},
 'private': False,
 'pulls_url': 'https://api.github.com/repos/vmware/pyvco/pulls{/number}',
 'pushed_at': '2010-11-07T15:55:51Z',
 'releases_url': 'https://api.github.com/repos/vmware/pyvco/releases{/id}',
 'size': 225,
 'ssh_url': 'git@github.com:vmware/pyvco.git',
 'stargazers_count': 10,
 'stargazers_url': 'https://api.github.com/repos/vmware/pyvco/stargazers',
 'statuses_url': 'https://api.github.com/repos/vmware/pyvco/statuses/{sha}',
 'subscribers_url': 'https://api.github.com/repos/vmware/pyvco/subscribers',
 'subscription_url': 'https://api.github.com/repos/vmware/pyvco/subscription',
 'svn_url': 'https://github.com/vmware/pyvco',
 'tags_url': 'https://api.github.com/repos/vmware/pyvco/tags',
 'teams_url': 'https://api.github.com/repos/vmware/pyvco/teams',
 'trees_url': 'https://api.github.com/repos/vmware/pyvco/git/trees{/sha}',
 'updated_at': '2018-03-02T08:41:42Z',
 'url': 'https://api.github.com/repos/vmware/pyvco',
 'watchers': 10,
 'watchers_count': 10}

for repo in repos:
    print(repo['full_name'], repo['forks'])

vmware/pyvco 3
vmware/rvc 48
vmware/rbvmomi 161
vmware/vprobe-toolkit 9
vmware/CloudFS 17
vmware/vcd-nclient 2
vmware/lmock 6
vmware/FireBreath 2
vmware/weasel 1
vmware/vmware-vcenter 95
vmware/vmware-vshield 6
vmware/vcloud-rest 37
vmware/GemstoneWebTools 0
vmware/vmware-vcsa 18
vmware/vmware-vmware_lib 27
vmware/saml20serviceprovider 2
vmware/pg_rewind 18
vmware/vco-powershel-plugin 2
vmware/jenkins-reviewbot 13
vmware/dbeekeeper 0
vmware/thinapp_factory 17
vmware/vmware-cassandra 4
vmware/vmware-java 0
vmware/data-driven-framework 4
vmware/pyvmomi 551
vmware/pyvmomi-community-samples 495
vmware/open-vm-tools 189
vmware/pyvmomi-tools 21
vmware/upgrade-framework 10
vmware/webcommander 34

def get_forks(r):
    return r['forks']
for repo in sorted(repos, key=get_forks):
    print(repo['full_name'], repo['forks'])

vmware/GemstoneWebTools 0
vmware/dbeekeeper 0
vmware/vmware-java 0
vmware/weasel 1
vmware/vcd-nclient 2
vmware/FireBreath 2
vmware/saml20serviceprovider 2
vmware/vco-powershel-plugin 2
vmware/pyvco 3
vmware/vmware-cassandra 4
vmware/data-driven-framework 4
vmware/lmock 6
vmware/vmware-vshield 6
vmware/vprobe-toolkit 9
vmware/upgrade-framework 10
vmware/jenkins-reviewbot 13
vmware/CloudFS 17
vmware/thinapp_factory 17
vmware/vmware-vcsa 18
vmware/pg_rewind 18
vmware/pyvmomi-tools 21
vmware/vmware-vmware_lib 27
vmware/webcommander 34
vmware/vcloud-rest 37
vmware/rvc 48
vmware/vmware-vcenter 95
vmware/rbvmomi 161
vmware/open-vm-tools 189
vmware/pyvmomi-community-samples 495
vmware/pyvmomi 551

for repo in sorted(repos, key=get_forks, reverse=True):
    print(repo['full_name'], repo['forks'])

vmware/pyvmomi 551
vmware/pyvmomi-community-samples 495
vmware/open-vm-tools 189
vmware/rbvmomi 161
vmware/vmware-vcenter 95
vmware/rvc 48
vmware/vcloud-rest 37
vmware/webcommander 34
vmware/vmware-vmware_lib 27
vmware/pyvmomi-tools 21
vmware/vmware-vcsa 18
vmware/pg_rewind 18
vmware/CloudFS 17
vmware/thinapp_factory 17
vmware/jenkins-reviewbot 13
vmware/upgrade-framework 10
vmware/vprobe-toolkit 9
vmware/lmock 6
vmware/vmware-vshield 6
vmware/vmware-cassandra 4
vmware/data-driven-framework 4
vmware/pyvco 3
vmware/vcd-nclient 2
vmware/FireBreath 2
vmware/saml20serviceprovider 2
vmware/vco-powershel-plugin 2
vmware/weasel 1
vmware/GemstoneWebTools 0
vmware/dbeekeeper 0
vmware/vmware-java 0

for repo in sorted(repos, key=get_forks, reverse=True)[:5]:
    print(repo['full_name'], repo['forks'])

vmware/pyvmomi 551
vmware/pyvmomi-community-samples 495
vmware/open-vm-tools 189
vmware/rbvmomi 161
vmware/vmware-vcenter 95

Executing on remote machine

from fabric2 import Connection

c = Connection("localhost")

r = c.run("ls")

AnacondaProjects
bin
Desktop
Documents
Downloads
foo-1.0
Music
Pictures
programming
Public
savings.org
Templates
trainings
usr
Videos
VirtualBox VMs
website.odg

r.return_code

0

from fabric2 import SerialGroup

g = SerialGroup("localhost", "localhost", "localhost")

r = g.run("ls")

AnacondaProjects
bin
Desktop
Documents
Downloads
foo-1.0
Music
Pictures
programming
Public
savings.org
Templates
trainings
usr
Videos
VirtualBox VMs
website.odg
AnacondaProjects
bin
Desktop
Documents
Downloads
foo-1.0
Music
Pictures
programming
Public
savings.org
Templates
trainings
usr
Videos
VirtualBox VMs
website.odg
AnacondaProjects
bin
Desktop
Documents
Downloads
foo-1.0
Music
Pictures
programming
Public
savings.org
Templates
trainings
usr
Videos
VirtualBox VMs
website.odg

type(r)

fabric2.group.GroupResult

help(Connection)

Help on class Connection in module fabric2.connection:

class Connection(invoke.context.Context)
 |  A connection to an SSH daemon, with methods for commands and file transfer.
 |  
 |  **Basics**
 |  
 |  This class inherits from Invoke's `~invoke.context.Context`, as it is a
 |  context within which commands, tasks etc can operate. It also encapsulates
 |  a Paramiko `~paramiko.client.SSHClient` instance, performing useful high
 |  level operations with that `~paramiko.client.SSHClient` and
 |  `~paramiko.channel.Channel` instances generated from it.
 |  
 |  **Lifecycle**
 |  
 |  `.Connection` has a basic "`create <__init__>`, `connect/open <open>`, `do
 |  work <run>`, `disconnect/close <close>`" lifecycle:
 |  
 |  - `Instantiation <__init__>` imprints the object with its connection
 |    parameters (but does **not** actually initiate the network connection).
 |  
 |      - An alternate constructor exists for users :ref:`upgrading piecemeal
 |        from Fabric 1 <from-v1>`: `from_v1`
 |  
 |  - Methods like `run`, `get` etc automatically trigger a call to
 |    `open` if the connection is not active; users may of course call `open`
 |    manually if desired.
 |  - Connections do not always need to be explicitly closed; much of the
 |    time, Paramiko's garbage collection hooks or Python's own shutdown
 |    sequence will take care of things. **However**, should you encounter edge
 |    cases (for example, sessions hanging on exit) it's helpful to explicitly
 |    close connections when you're done with them.
 |  
 |    This can be accomplished by manually calling `close`, or by using the
 |    object as a contextmanager::
 |  
 |      with Connection('host') as c:
 |          c.run('command')
 |          c.put('file')
 |  
 |  .. note::
 |      This class rebinds `invoke.context.Context.run` to `.local` so both
 |      remote and local command execution can coexist.
 |  
 |  **Configuration**
 |  
 |  Most `.Connection` parameters honor :doc:`Invoke-style configuration
 |  </concepts/configuration>` as well as any applicable :ref:`SSH config file
 |  directives <connection-ssh-config>`. For example, to end up with a
 |  connection to ``admin@myhost``, one could:
 |  
 |  - Use any built-in config mechanism, such as ``/etc/fabric.yml``,
 |    ``~/.fabric.json``, collection-driven configuration, env vars, etc,
 |    stating ``user: admin`` (or ``{"user": "admin"}``, depending on config
 |    format.) Then ``Connection('myhost')`` would implicitly have a ``user``
 |    of ``admin``.
 |  - Use an SSH config file containing ``User admin`` within any applicable
 |    ``Host`` header (``Host myhost``, ``Host *``, etc.) Again,
 |    ``Connection('myhost')`` will default to an ``admin`` user.
 |  - Leverage host-parameter shorthand (described in `.Config.__init__`), i.e.
 |    ``Connection('admin@myhost')``.
 |  - Give the parameter directly: ``Connection('myhost', user='admin')``.
 |  
 |  The same applies to agent forwarding, gateways, and so forth.
 |  
 |  .. versionadded:: 2.0
 |  
 |  Method resolution order:
 |      Connection
 |      invoke.context.Context
 |      invoke.config.DataProxy
 |      builtins.object
 |  
 |  Methods defined here:
 |  
 |  __enter__(self)
 |  
 |  __eq__(self, other)
 |      Return self==value.
 |  
 |  __exit__(self, *exc)
 |  
 |  __hash__(self)
 |      Return hash(self).
 |  
 |  __init__(self, host, user=None, port=None, config=None, gateway=None, forward_agent=None, connect_timeout=None, connect_kwargs=None, inline_ssh_env=None)
 |      Set up a new object representing a server connection.
 |      
 |      :param str host:
 |          the hostname (or IP address) of this connection.
 |      
 |          May include shorthand for the ``user`` and/or ``port`` parameters,
 |          of the form ``user@host``, ``host:port``, or ``user@host:port``.
 |      
 |          .. note::
 |              Due to ambiguity, IPv6 host addresses are incompatible with the
 |              ``host:port`` shorthand (though ``user@host`` will still work
 |              OK). In other words, the presence of >1 ``:`` character will
 |              prevent any attempt to derive a shorthand port number; use the
 |              explicit ``port`` parameter instead.
 |      
 |          .. note::
 |              If ``host`` matches a ``Host`` clause in loaded SSH config
 |              data, and that ``Host`` clause contains a ``Hostname``
 |              directive, the resulting `.Connection` object will behave as if
 |              ``host`` is equal to that ``Hostname`` value.
 |      
 |              In all cases, the original value of ``host`` is preserved as
 |              the ``original_host`` attribute.
 |      
 |              Thus, given SSH config like so::
 |      
 |                  Host myalias
 |                      Hostname realhostname
 |      
 |              a call like ``Connection(host='myalias')`` will result in an
 |              object whose ``host`` attribute is ``realhostname``, and whose
 |              ``original_host`` attribute is ``myalias``.
 |      
 |      :param str user:
 |          the login user for the remote connection. Defaults to
 |          ``config.user``.
 |      
 |      :param int port:
 |          the remote port. Defaults to ``config.port``.
 |      
 |      :param config:
 |          configuration settings to use when executing methods on this
 |          `.Connection` (e.g. default SSH port and so forth).
 |      
 |          Should be a `.Config` or an `invoke.config.Config`
 |          (which will be turned into a `.Config`).
 |      
 |          Default is an anonymous `.Config` object.
 |      
 |      :param gateway:
 |          An object to use as a proxy or gateway for this connection.
 |      
 |          This parameter accepts one of the following:
 |      
 |          - another `.Connection` (for a ``ProxyJump`` style gateway);
 |          - a shell command string (for a ``ProxyCommand`` style style
 |            gateway).
 |      
 |          Default: ``None``, meaning no gatewaying will occur (unless
 |          otherwise configured; if one wants to override a configured gateway
 |          at runtime, specify ``gateway=False``.)
 |      
 |          .. seealso:: :ref:`ssh-gateways`
 |      
 |      :param bool forward_agent:
 |          Whether to enable SSH agent forwarding.
 |      
 |          Default: ``config.forward_agent``.
 |      
 |      :param int connect_timeout:
 |          Connection timeout, in seconds.
 |      
 |          Default: ``config.timeouts.connect``.
 |      
 |      :param dict connect_kwargs:
 |          Keyword arguments handed verbatim to
 |          `SSHClient.connect <paramiko.client.SSHClient.connect>` (when
 |          `.open` is called).
 |      
 |          `.Connection` tries not to grow additional settings/kwargs of its
 |          own unless it is adding value of some kind; thus,
 |          ``connect_kwargs`` is currently the right place to hand in
 |          parameters such as ``pkey`` or ``key_filename``.
 |      
 |          Default: ``config.connect_kwargs``.
 |      
 |      :param bool inline_ssh_env:
 |          Whether to send environment variables "inline" as prefixes in front
 |          of command strings (``export VARNAME=value && mycommand here``),
 |          instead of trying to submit them through the SSH protocol itself
 |          (which is the default behavior). This is necessary if the remote
 |          server has a restricted ``AcceptEnv`` setting (which is the common
 |          default).
 |      
 |          The default value is the value of the ``inline_ssh_env``
 |          :ref:`configuration value <default-values>` (which itself defaults
 |          to ``False``).
 |      
 |          .. warning::
 |              This functionality does **not** currently perform any shell
 |              escaping on your behalf! Be careful when using nontrivial
 |              values, and note that you can put in your own quoting,
 |              backslashing etc if desired.
 |      
 |              Consider using a different approach (such as actual
 |              remote shell scripts) if you run into too many issues here.
 |      
 |          .. note::
 |              When serializing into prefixed ``FOO=bar`` format, we apply the
 |              builtin `sorted` function to the env dictionary's keys, to
 |              remove what would otherwise be ambiguous/arbitrary ordering.
 |      
 |          .. note::
 |              This setting has no bearing on *local* shell commands; it only
 |              affects remote commands, and thus, methods like `.run` and
 |              `.sudo`.
 |      
 |      :raises ValueError:
 |          if user or port values are given via both ``host`` shorthand *and*
 |          their own arguments. (We `refuse the temptation to guess`_).
 |      
 |      .. _refuse the temptation to guess:
 |          http://zen-of-python.info/
 |          in-the-face-of-ambiguity-refuse-the-temptation-to-guess.html#12
 |      
 |      .. versionchanged:: 2.3
 |          Added the ``inline_ssh_env`` parameter.
 |  
 |  __lt__(self, other)
 |      Return self<value.
 |  
 |  __repr__(self)
 |      Return repr(self).
 |  
 |  close(self)
 |      Terminate the network connection to the remote end, if open.
 |      
 |      If no connection is open, this method does nothing.
 |      
 |      .. versionadded:: 2.0
 |  
 |  create_session(self)
 |  
 |  derive_shorthand(self, host_string)
 |  
 |  forward_local(self, local_port, remote_port=None, remote_host='localhost', local_host='localhost')
 |      Open a tunnel connecting ``local_port`` to the server's environment.
 |      
 |      For example, say you want to connect to a remote PostgreSQL database
 |      which is locked down and only accessible via the system it's running
 |      on. You have SSH access to this server, so you can temporarily make
 |      port 5432 on your local system act like port 5432 on the server::
 |      
 |          import psycopg2
 |          from fabric import Connection
 |      
 |          with Connection('my-db-server').forward_local(5432):
 |              db = psycopg2.connect(
 |                  host='localhost', port=5432, database='mydb'
 |              )
 |              # Do things with 'db' here
 |      
 |      This method is analogous to using the ``-L`` option of OpenSSH's
 |      ``ssh`` program.
 |      
 |      :param int local_port: The local port number on which to listen.
 |      
 |      :param int remote_port:
 |          The remote port number. Defaults to the same value as
 |          ``local_port``.
 |      
 |      :param str local_host:
 |          The local hostname/interface on which to listen. Default:
 |          ``localhost``.
 |      
 |      :param str remote_host:
 |          The remote hostname serving the forwarded remote port. Default:
 |          ``localhost`` (i.e., the host this `.Connection` is connected to.)
 |      
 |      :returns:
 |          Nothing; this method is only useful as a context manager affecting
 |          local operating system state.
 |      
 |      .. versionadded:: 2.0
 |  
 |  forward_remote(self, remote_port, local_port=None, remote_host='127.0.0.1', local_host='localhost')
 |      Open a tunnel connecting ``remote_port`` to the local environment.
 |      
 |      For example, say you're running a daemon in development mode on your
 |      workstation at port 8080, and want to funnel traffic to it from a
 |      production or staging environment.
 |      
 |      In most situations this isn't possible as your office/home network
 |      probably blocks inbound traffic. But you have SSH access to this
 |      server, so you can temporarily make port 8080 on that server act like
 |      port 8080 on your workstation::
 |      
 |          from fabric import Connection
 |      
 |          c = Connection('my-remote-server')
 |          with c.forward_remote(8080):
 |              c.run("remote-data-writer --port 8080")
 |              # Assuming remote-data-writer runs until interrupted, this will
 |              # stay open until you Ctrl-C...
 |      
 |      This method is analogous to using the ``-R`` option of OpenSSH's
 |      ``ssh`` program.
 |      
 |      :param int remote_port: The remote port number on which to listen.
 |      
 |      :param int local_port:
 |          The local port number. Defaults to the same value as
 |          ``remote_port``.
 |      
 |      :param str local_host:
 |          The local hostname/interface the forwarded connection talks to.
 |          Default: ``localhost``.
 |      
 |      :param str remote_host:
 |          The remote interface address to listen on when forwarding
 |          connections. Default: ``127.0.0.1`` (i.e. only listen on the remote
 |          localhost).
 |      
 |      :returns:
 |          Nothing; this method is only useful as a context manager affecting
 |          local operating system state.
 |      
 |      .. versionadded:: 2.0
 |  
 |  get(self, *args, **kwargs)
 |      Get a remote file to the local filesystem or file-like object.
 |      
 |      Simply a wrapper for `.Transfer.get`. Please see its documentation for
 |      all details.
 |      
 |      .. versionadded:: 2.0
 |  
 |  get_gateway(self)
 |  
 |  local(self, *args, **kwargs)
 |      Execute a shell command on the local system.
 |      
 |      This method is effectively a wrapper of `invoke.run`; see its docs for
 |      details and call signature.
 |      
 |      .. versionadded:: 2.0
 |  
 |  open(self)
 |      Initiate an SSH connection to the host/port this object is bound to.
 |      
 |      This may include activating the configured gateway connection, if one
 |      is set.
 |      
 |      Also saves a handle to the now-set Transport object for easier access.
 |      
 |      Various connect-time settings (and/or their corresponding :ref:`SSH
 |      config options <ssh-config>`) are utilized here in the call to
 |      `SSHClient.connect <paramiko.client.SSHClient.connect>`. (For details,
 |      see :doc:`the configuration docs </concepts/configuration>`.)
 |      
 |      .. versionadded:: 2.0
 |  
 |  open_gateway(self)
 |      Obtain a socket-like object from `gateway`.
 |      
 |      :returns:
 |          A ``direct-tcpip`` `paramiko.channel.Channel`, if `gateway` was a
 |          `.Connection`; or a `~paramiko.proxy.ProxyCommand`, if `gateway`
 |          was a string.
 |      
 |      .. versionadded:: 2.0
 |  
 |  put(self, *args, **kwargs)
 |      Put a remote file (or file-like object) to the remote filesystem.
 |      
 |      Simply a wrapper for `.Transfer.put`. Please see its documentation for
 |      all details.
 |      
 |      .. versionadded:: 2.0
 |  
 |  resolve_connect_kwargs(self, connect_kwargs)
 |  
 |  run(self, command, **kwargs)
 |      Execute a shell command on the remote end of this connection.
 |      
 |      This method wraps an SSH-capable implementation of
 |      `invoke.runners.Runner.run`; see its documentation for details.
 |      
 |      .. warning::
 |          There are a few spots where Fabric departs from Invoke's default
 |          settings/behaviors; they are documented under
 |          `.Config.global_defaults`.
 |      
 |      .. versionadded:: 2.0
 |  
 |  sftp(self)
 |      Return a `~paramiko.sftp_client.SFTPClient` object.
 |      
 |      If called more than one time, memoizes the first result; thus, any
 |      given `.Connection` instance will only ever have a single SFTP client,
 |      and state (such as that managed by
 |      `~paramiko.sftp_client.SFTPClient.chdir`) will be preserved.
 |      
 |      .. versionadded:: 2.0
 |  
 |  sudo(self, command, **kwargs)
 |      Execute a shell command, via ``sudo``, on the remote end.
 |      
 |      This method is identical to `invoke.context.Context.sudo` in every way,
 |      except in that -- like `run` -- it honors per-host/per-connection
 |      configuration overrides in addition to the generic/global ones. Thus,
 |      for example, per-host sudo passwords may be configured.
 |      
 |      .. versionadded:: 2.0
 |  
 |  ----------------------------------------------------------------------
 |  Class methods defined here:
 |  
 |  from_v1(env, **kwargs) from builtins.type
 |      Alternate constructor which uses Fabric 1's ``env`` dict for settings.
 |      
 |      All keyword arguments besides ``env`` are passed unmolested into the
 |      primary constructor.
 |      
 |      .. warning::
 |          Because your own config overrides will win over data from ``env``,
 |          make sure you only set values you *intend* to change from your v1
 |          environment!
 |      
 |      For details on exactly which ``env`` vars are imported and what they
 |      become in the new API, please see :ref:`v1-env-var-imports`.
 |      
 |      :param env:
 |          An explicit Fabric 1 ``env`` dict (technically, any
 |          ``fabric.utils._AttributeDict`` instance should work) to pull
 |          configuration from.
 |      
 |      .. versionadded:: 2.4
 |  
 |  ----------------------------------------------------------------------
 |  Data descriptors defined here:
 |  
 |  is_connected
 |      Whether or not this connection is actually open.
 |      
 |      .. versionadded:: 2.0
 |  
 |  ----------------------------------------------------------------------
 |  Data and other attributes defined here:
 |  
 |  client = None
 |  
 |  connect_kwargs = None
 |  
 |  connect_timeout = None
 |  
 |  forward_agent = None
 |  
 |  gateway = None
 |  
 |  host = None
 |  
 |  original_host = None
 |  
 |  port = None
 |  
 |  ssh_config = None
 |  
 |  transport = None
 |  
 |  user = None
 |  
 |  ----------------------------------------------------------------------
 |  Methods inherited from invoke.context.Context:
 |  
 |  cd(self, path)
 |      Context manager that keeps directory state when executing commands.
 |      
 |      Any calls to `run`, `sudo`, within the wrapped block will implicitly
 |      have a string similar to ``"cd <path> && "`` prefixed in order to give
 |      the sense that there is actually statefulness involved.
 |      
 |      Because use of `cd` affects all such invocations, any code making use
 |      of the `cwd` property will also be affected by use of `cd`.
 |      
 |      Like the actual 'cd' shell builtin, `cd` may be called with relative
 |      paths (keep in mind that your default starting directory is your user's
 |      ``$HOME``) and may be nested as well.
 |      
 |      Below is a "normal" attempt at using the shell 'cd', which doesn't work
 |      since all commands are executed in individual subprocesses -- state is
 |      **not** kept between invocations of `run` or `sudo`::
 |      
 |          c.run('cd /var/www')
 |          c.run('ls')
 |      
 |      The above snippet will list the contents of the user's ``$HOME``
 |      instead of ``/var/www``. With `cd`, however, it will work as expected::
 |      
 |          with c.cd('/var/www'):
 |              c.run('ls')  # Turns into "cd /var/www && ls"
 |      
 |      Finally, a demonstration (see inline comments) of nesting::
 |      
 |          with c.cd('/var/www'):
 |              c.run('ls') # cd /var/www && ls
 |              with c.cd('website1'):
 |                  c.run('ls')  # cd /var/www/website1 && ls
 |      
 |      .. note::
 |          Space characters will be escaped automatically to make dealing with
 |          such directory names easier.
 |      
 |      .. versionadded:: 1.0
 |  
 |  prefix(self, command)
 |      Prefix all nested `run`/`sudo` commands with given command plus ``&&``.
 |      
 |      Most of the time, you'll want to be using this alongside a shell script
 |      which alters shell state, such as ones which export or alter shell
 |      environment variables.
 |      
 |      For example, one of the most common uses of this tool is with the
 |      ``workon`` command from `virtualenvwrapper
 |      <https://virtualenvwrapper.readthedocs.io/en/latest/>`_::
 |      
 |          with c.prefix('workon myvenv'):
 |              c.run('./manage.py migrate')
 |      
 |      In the above snippet, the actual shell command run would be this::
 |      
 |          $ workon myvenv && ./manage.py migrate
 |      
 |      This context manager is compatible with `cd`, so if your virtualenv
 |      doesn't ``cd`` in its ``postactivate`` script, you could do the
 |      following::
 |      
 |          with c.cd('/path/to/app'):
 |              with c.prefix('workon myvenv'):
 |                  c.run('./manage.py migrate')
 |                  c.run('./manage.py loaddata fixture')
 |      
 |      Which would result in executions like so::
 |      
 |          $ cd /path/to/app && workon myvenv && ./manage.py migrate
 |          $ cd /path/to/app && workon myvenv && ./manage.py loaddata fixture
 |      
 |      Finally, as alluded to above, `prefix` may be nested if desired, e.g.::
 |      
 |          with c.prefix('workon myenv'):
 |              c.run('ls')
 |              with c.prefix('source /some/script'):
 |                  c.run('touch a_file')
 |      
 |      The result::
 |      
 |          $ workon myenv && ls
 |          $ workon myenv && source /some/script && touch a_file
 |      
 |      Contrived, but hopefully illustrative.
 |      
 |      .. versionadded:: 1.0
 |  
 |  ----------------------------------------------------------------------
 |  Data descriptors inherited from invoke.context.Context:
 |  
 |  config
 |  
 |  cwd
 |      Return the current working directory, accounting for uses of `cd`.
 |      
 |      .. versionadded:: 1.0
 |  
 |  ----------------------------------------------------------------------
 |  Methods inherited from invoke.config.DataProxy:
 |  
 |  __contains__(self, key)
 |  
 |  __delattr__(self, name)
 |      Implement delattr(self, name).
 |  
 |  __delitem__(self, key)
 |  
 |  __getattr__(self, key)
 |  
 |  __getitem__(self, key)
 |  
 |  __iter__(self)
 |  
 |  __len__(self)
 |  
 |  __setattr__(self, key, value)
 |      Implement setattr(self, name, value).
 |  
 |  __setitem__(self, key, value)
 |  
 |  clear(self)
 |  
 |  pop(self, *args)
 |  
 |  popitem(self)
 |  
 |  setdefault(self, *args)
 |  
 |  update(self, *args, **kwargs)
 |  
 |  ----------------------------------------------------------------------
 |  Class methods inherited from invoke.config.DataProxy:
 |  
 |  from_data(data, root=None, keypath=()) from builtins.type
 |      Alternate constructor for 'baby' DataProxies used as sub-dict values.
 |      
 |      Allows creating standalone DataProxy objects while also letting
 |      subclasses like `.Config` define their own ``__init__`` without
 |      muddling the two.
 |      
 |      :param dict data:
 |          This particular DataProxy's personal data. Required, it's the Data
 |          being Proxied.
 |      
 |      :param root:
 |          Optional handle on a root DataProxy/Config which needs notification
 |          on data updates.
 |      
 |      :param tuple keypath:
 |          Optional tuple describing the path of keys leading to this
 |          DataProxy's location inside the ``root`` structure. Required if
 |          ``root`` was given (and vice versa.)
 |      
 |      .. versionadded:: 1.0
 |  
 |  ----------------------------------------------------------------------
 |  Data descriptors inherited from invoke.config.DataProxy:
 |  
 |  __dict__
 |      dictionary for instance variables (if defined)
 |  
 |  __weakref__
 |      list of weak references to the object (if defined)

from invoke import Responder

references

• Python docs
• Python cookbook by David Beazley and others
• David Beazley talk on Concurrency from ground up.
• attend python conferences in india
• bangpypers group.
• basic "python practice book" by Anand Citipothu
• SICP