Python Virtual Training For Arcesium - Module I - Day 5

Sep 13-17, 2021 Vikrant Patil

These notes are available online at https://notes.pipal.in/2021/arcesium_finop_batch1/module1-day5.html

© Pipal Academy LLP

Day 1 | Day 2 | Day 3 | Day 4 | Day 5

We will be using jupyter hub from https://lab.pipal.in for this training.

login to hub and create a notebook with name module1-day5

• List of urls is given. Some urls are from same domain, some are from different. Find unique domain names used in the urls.

     urls = ['www.abrakadabra.com/dccEcB/EGdd',
    'www.abrakadabra.com/gADFeD/bcAF',
    'www.abra.com/AGadbb/eagE',
    'www.dabra.com/cffdfD/FCAD',
    'www.abra.com/GFGaBE/dcfc',
    'www.abra.com/gaFegG/Bdaf',
    'www.abrakadabra.com/aGabaf/EEfa',
    'www.dabra.com/ceEgFD/bGgc',
    'www.dabra.com/bDEffC/bcEA']

• Write a function min2 which find minimum from given two numbers. Also write a function min3 which can find minimum number from given 3 numbers. Do not make use of bulit in min function, make use of if , elif/else blocks.

>>> min2(3, 4)
3
>>> min3(4, 2, 6)
2

def unique(items):
    unique_ = []
    
    for item in items:
        if item not in unique_:
            unique_.append(item)
            
    return unique_

unique([1, 1, 1, 2, 3, 2, 3, 5, 5])

[1, 2, 3, 5]

def domains(urls):
    d = []
    for url in urls:
        domain = url.split("/")[0]
        d.append(domain)
        
    return d
    

urls = ['www.abrakadabra.com/dccEcB/EGdd',
    'www.abrakadabra.com/gADFeD/bcAF',
    'www.abra.com/AGadbb/eagE',
    'www.dabra.com/cffdfD/FCAD',
    'www.abra.com/GFGaBE/dcfc',
    'www.abra.com/gaFegG/Bdaf',
    'www.abrakadabra.com/aGabaf/EEfa',
    'www.dabra.com/ceEgFD/bGgc',
    'www.dabra.com/bDEffC/bcEA']

domains(urls)

['www.abrakadabra.com',
 'www.abrakadabra.com',
 'www.abra.com',
 'www.dabra.com',
 'www.abra.com',
 'www.abra.com',
 'www.abrakadabra.com',
 'www.dabra.com',
 'www.dabra.com']

unique(domains(urls))

['www.abrakadabra.com', 'www.abra.com', 'www.dabra.com']

def min2(x, y):
    if x < y:
        return x
    else:
        return y

min2(5, 7)

5

def min3(x, y, z):
    if x < y and x < z:
        return x
    elif y < x and y < z:
        return y
    else:
        return z

min3(5, 6, 4)

4

3, 5, 12, 7, 3, 2, 11, 10

min    next comparison
3       there is nothing compare for first element
3       5
3       12
3       7
3       3
3       2
2       11
2       10
2

def min3(x, y, z):
    m = min2(x, y)
    return min2(m, z)

def min3(x, y, z):
    return min2(min2(x, y), z)

def min_(*args): # * before any name means that it is variable number arguments
    pass # empty statement


def min_(*nums):
    print(nums)

min_(1, 2)

(1, 2)

min_(2, 3, 4, 5, 6)

(2, 3, 4, 5, 6)

def min_(*nums):
    m = nums[0] # this is first element
    for n in nums[1:]: # drop frist 1 element
        m = min2(m, n)
        
    return m

min_(3, 6, 2, 6, 1, 4, 5)

1

def foo(fixed, *args):
    print(fixed)
    print(args)

foo("hello", 3, 4, 5, 6)

hello
(3, 4, 5, 6)

foo("hello", 1 , 2)

hello
(1, 2)

def foo(*args, fixed): # this is not allowed
    print(fixed)
    print(args)

foo(2, 3, 4, "hello")

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-29-fb12a43e32cd> in <module>
----> 1 foo(2, 3, 4, "hello")

TypeError: foo() missing 1 required keyword-only argument: 'fixed'

Modules and Scripts

with functions we commbined multiple statements together , gave it name. recalled it with function call

import os # importing python's built in modules

os

<module 'os' from '/home/vikrant/anaconda3/lib/python3.8/os.py'>

print(os)

<module 'os' from '/home/vikrant/anaconda3/lib/python3.8/os.py'>

os.getcwd() # this will return current working directory

'/home/vikrant/trainings/2021/arcesium_finop_batch1'

currentpath = os.getcwd()
os.path.isdir(currentpath) # this will check if the path that is passed as argument is folder or not

True

os.path.isfile(currentpath)

False

os.listdir() # give me list of all files in current working directory

['index.html',
 'problems-day4.html',
 'module1-day5.html',
 'problems-day2.html',
 'push',
 'module1-day5.ipynb',
 'problems-day3.ipynb',
 'Untitled.html',
 'problems-day3.html',
 'index.ipynb',
 'problems-day4.ipynb',
 'module1-day1.html',
 'module1-day1.ipynb',
 'module1-day2.ipynb',
 'module1-day3.html',
 'module1-day2.html',
 'module1-day3.ipynb',
 'backup',
 'module1-day4.html',
 'Makefile',
 '.ipynb_checkpoints',
 'module1-day4.ipynb',
 'problems-day2.ipynb']

os.listdir("/etc/ssh/")# this will show all files and directories in "/etc/ssh"

['ssh_host_ed25519_key',
 'ssh_host_rsa_key.pub',
 'ssh_host_ecdsa_key.pub',
 'ssh_config.d',
 'ssh_host_ecdsa_key',
 'sshd_config',
 'ssh_host_ed25519_key.pub',
 'ssh_host_rsa_key',
 'ssh_import_id',
 'moduli',
 'ssh_config',
 'sshd_config.d']

os.listdir() # if no argument is passed, it will take current working directory

['index.html',
 'problems-day4.html',
 'module1-day5.html',
 'problems-day2.html',
 'push',
 'module1-day5.ipynb',
 'problems-day3.ipynb',
 'Untitled.html',
 'problems-day3.html',
 'index.ipynb',
 'problems-day4.ipynb',
 'module1-day1.html',
 'module1-day1.ipynb',
 'module1-day2.ipynb',
 'module1-day3.html',
 'module1-day2.html',
 'module1-day3.ipynb',
 'backup',
 'module1-day4.html',
 'Makefile',
 '.ipynb_checkpoints',
 'module1-day4.ipynb',
 'problems-day2.ipynb']

os.path.sep

'/'

os.path.sep.join([os.getcwd(), "index.html"])

'/home/vikrant/trainings/2021/arcesium_finop_batch1/index.html'

indexfile = os.path.sep.join([os.getcwd(), "index.html"])

indexfile

'/home/vikrant/trainings/2021/arcesium_finop_batch1/index.html'

os.path.isdir(indexfile)

False

os.path.isfile(indexfile)

True

def get_folders(folderpath):
    """gets all folders under given folder
    """
    listing = os.listdir(folderpath) ## all the files and folders from folderpath
    
    folders = []
    
    for item in listing:
        completepath = os.path.sep.join([folderpath, item])
        if os.path.isdir(completepath):
            folders.append(item)
            
    return folders

get_folders(os.getcwd())

['backup', '.ipynb_checkpoints']

+ /home/vikrant
  + trainings
  + programming
  + Pictures
  +
  somefile.txt
  hello.py

complete path of hello.py is /home/vikrant/hello.py

os.path.isdir("/home/vikrant/hello.py") -> False

":".join(["hello", "world"])

'hello:world'

"/".join(["/home/vikrant", "hello.py"])

'/home/vikrant/hello.py'

os.path.sep

'/'

def get_files(folderpath):
    """gets all folders under given folder
    """
    listing = os.listdir(folderpath) ## all the files and folders from folderpath
    
    files = []
    
    for item in listing:
        completepath = os.path.sep.join([folderpath, item])
        if os.path.isfile(completepath): # changed thois check
            files.append(item)
            
    return files

get_files(os.getcwd())

['index.html',
 'problems-day4.html',
 'module1-day5.html',
 'problems-day2.html',
 'push',
 'module1-day5.ipynb',
 'problems-day3.ipynb',
 'Untitled.html',
 'problems-day3.html',
 'index.ipynb',
 'problems-day4.ipynb',
 'module1-day1.html',
 'module1-day1.ipynb',
 'module1-day2.ipynb',
 'module1-day3.html',
 'module1-day2.html',
 'module1-day3.ipynb',
 'module1-day4.html',
 'Makefile',
 'module1-day4.ipynb',
 'problems-day2.ipynb']

os.path.isfile("index.html")

True

get_files("/etc/ssh")

['ssh_host_ed25519_key',
 'ssh_host_rsa_key.pub',
 'ssh_host_ecdsa_key.pub',
 'ssh_host_ecdsa_key',
 'sshd_config',
 'ssh_host_ed25519_key.pub',
 'ssh_host_rsa_key',
 'ssh_import_id',
 'moduli',
 'ssh_config']

os.path.isfile('ssh_config') # this will return False because although ssh_config is a file ... it is not in current working directory

False

os.path.isfile(os.path.sep.join(["/etc/ssh", 'ssh_config']))  # because complete path is given

True

os.listdir("/etc/ssh") # this will given everything which includes folders and files

['ssh_host_ed25519_key',
 'ssh_host_rsa_key.pub',
 'ssh_host_ecdsa_key.pub',
 'ssh_config.d',
 'ssh_host_ecdsa_key',
 'sshd_config',
 'ssh_host_ed25519_key.pub',
 'ssh_host_rsa_key',
 'ssh_import_id',
 'moduli',
 'ssh_config',
 'sshd_config.d']

os.path.getsize("index.html") # gives size in bytes

569051

os.path.getsize("ssh_config")

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-68-6f6ae7741d89> in <module>
----> 1 os.path.getsize("ssh_config")

~/anaconda3/lib/python3.8/genericpath.py in getsize(filename)
     48 def getsize(filename):
     49     """Return the size of a file, reported by os.stat()."""
---> 50     return os.stat(filename).st_size
     51 
     52 

FileNotFoundError: [Errno 2] No such file or directory: 'ssh_config'

completepaht_ssh_config = os.path.isfile(os.path.sep.join(["/etc/ssh", 'ssh_config']))
os.path.getsize(completepaht_ssh_config)

0

problem

• Can you find biggest file (content size) in your current working directory?

os.getcwd()

'/home/vikrant/trainings/2021/arcesium_finop_batch1'

os.listdir("index.html")

---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-71-192b5fb000b8> in <module>
----> 1 os.listdir("index.html")

NotADirectoryError: [Errno 20] Not a directory: 'index.html'

words = "SOme words for testing".split()
max(words, key=len)

'testing'

max(os.listdir(), key=os.path.getsize)

'module1-day2.html'

for file in get_files(os.getcwd()):
    print(file.rjust(20), os.path.getsize(file))

          index.html 569051
  problems-day4.html 568997
   module1-day5.html 637378
  problems-day2.html 569101
                push 0
  module1-day5.ipynb 28159
 problems-day3.ipynb 1550
       Untitled.html 567514
  problems-day3.html 568964
         index.ipynb 1347
 problems-day4.ipynb 1553
   module1-day1.html 715596
  module1-day1.ipynb 54851
  module1-day2.ipynb 59146
   module1-day3.html 688433
   module1-day2.html 735468
  module1-day3.ipynb 47720
   module1-day4.html 718894
            Makefile 627
  module1-day4.ipynb 54902
 problems-day2.ipynb 1499

os.path.getsize(os.getcwd())

4096

for file in os.listdir():
    print(file.rjust(20), os.path.getsize(file))

          index.html 569051
  problems-day4.html 568997
   module1-day5.html 643045
  problems-day2.html 569101
                push 0
  module1-day5.ipynb 30317
 problems-day3.ipynb 1550
       Untitled.html 567514
  problems-day3.html 568964
         index.ipynb 1347
 problems-day4.ipynb 1553
   module1-day1.html 715596
  module1-day1.ipynb 54851
  module1-day2.ipynb 59146
   module1-day3.html 688433
   module1-day2.html 735468
  module1-day3.ipynb 47720
              backup 4096
   module1-day4.html 718894
            Makefile 627
  .ipynb_checkpoints 4096
  module1-day4.ipynb 54902
 problems-day2.ipynb 1499

import math as m

m

<module 'math' from '/home/vikrant/anaconda3/lib/python3.8/lib-dynload/math.cpython-38-x86_64-linux-gnu.so'>

import pandas as pd

pd.read_csv() #

from os import getcwd # here you will be importing only getcwd function

getcwd()

'/home/vikrant/trainings/2021/arcesium_finop_batch1'

os.listdir(os.path.sep.join(["/home/vikrant", ".profile"]))

---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-85-06d928f43711> in <module>
----> 1 os.listdir(os.path.sep.join(["/home/vikrant", ".profile"]))

NotADirectoryError: [Errno 20] Not a directory: '/home/vikrant/.profile'

Writing your own modules

import mymath

help(mymath)

Help on module mymath:

NAME
    mymath - This is my math module which has got some functions which are basic maths functions.

FUNCTIONS
    absolute(x)
    
    average(numbers)
    
    mysum(*args)
        sums variable number of arguments
    
    square(x)
        computes square of given number

FILE
    /home/vikrant/trainings/2021/arcesium_finop_batch1/mymath.py

mymath.mysum(3, 4, 5)

12

mymath.mysum(1, 2)

3

mymath.average([3,2, 43, 54, 657, 323])

180.33333333333334

mymath.absolute(54)

54

mymath.absolute(-55)

55

%%file mymodule.py
"""This is my first python module
which I have created from jupyter notebook
"""

def say_hello(name):
    print("Hello", name.capitalize())

Overwriting mymodule.py

import mymodule

help(mymodule)

Help on module mymodule:

NAME
    mymodule

DESCRIPTION
    This is my first python module
    which I have created from jupyter notebook

FUNCTIONS
    say_hello(name)

FILE
    /home/vikrant/trainings/2021/arcesium_finop_batch1/mymodule.py

mymodule.say_hello("vikrant")

Hello Vikrant

%%file mymodule.py
"""This is my first python module
which I have created from jupyter notebook
"""

def say_hello(name):
    print("Hello", name.capitalize())

    
def welcome(name):
    print("Hello ", name.capitalize(), "Welcome to mymodule")

Overwriting mymodule.py

import mymodule

mymodule.welcome("vikrant")

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-103-dff5a4835023> in <module>
----> 1 mymodule.welcome("vikrant")

AttributeError: module 'mymodule' has no attribute 'welcome'

%%file mymodule1.py
"""This is my first python module
which I have created from jupyter notebook
"""

def say_hello(name):
    print("Hello", name.capitalize())

    
def welcome(name):
    print("Hello ", name.capitalize(), "Welcome to mymodule")

Writing mymodule1.py

import mymodule1

import os # this is doing nothing because os module is already imported

def foo():
    
    print("foo")

foo()

foo

Python scripts

a program that can be run from command line.

python3 hello.py
Hello World!

%%file hello.py

print("Hello World!")

Writing hello.py

How to run commands from jupyter...

!python3 hello.py

!python3 hello.py

Hello World!

Command line arguments

How to pass command line arguments from command line?

%%file helloworld.py
import sys


sys.argv
print(sys.argv)


def say_hello(name):
    print("Hello", name.capitalize())


def welcome(name):
    print("Welcome", name.capitalize())


#welcome(sys.argv[1])
#say_hello(sys.argv[1])
#print("Hello I am running my first python program, ", sys.argv[0])

Overwriting helloworld.py

!python3 helloworld.py arg1

['helloworld.py', 'arg1']

!python3 helloworld.py arg1 arg2

['helloworld.py', 'arg1', 'arg2']

!python3 helloworld.py arg1 arg3

['helloworld.py', 'arg1', 'arg3']

%%file helloworld.py
import sys


#sys.argv
#print(sys.argv)


def say_hello(name):
    print("Hello", name.capitalize())


def welcome(name):
    print("Welcome", name.capitalize())


welcome(sys.argv[1])
say_hello(sys.argv[1])
print("Hello I am running my first python program, ", sys.argv[0])

Overwriting helloworld.py

!python3 helloworld.py vikrant

Welcome Vikrant
Hello Vikrant
Hello I am running my first python program,  helloworld.py

%%file square.py
import sys

def square(x):
    return x*x


square(sys.argv[1])# arguments are text by default

Writing square.py

!python square.py 5

Traceback (most recent call last):
  File "square.py", line 7, in <module>
    square(sys.argv[1])# arguments are text by default
  File "square.py", line 4, in square
    return x*x
TypeError: can't multiply sequence by non-int of type 'str'

%%file square.py
import sys

def square(x):
    return x*x


square(int(sys.argv[1]))# arguments are text by default

Overwriting square.py

!python3 square.py 5 # no results printed unless you print it

%%file square.py
import sys

def square(x):
    return x*x


s = square(int(sys.argv[1]))# arguments are text by default
print(s)

Overwriting square.py

!python3 square.py 5

25