Python Virtual Training For Arcesium - Module III

Oct 11-15, 2021 Vikrant Patil

These notes are available online at https://notes.pipal.in/2021/arcesium_finop_batch2/

© Pipal Academy LLP

Day 1 | Day 2 | Day 3 | Day 4 | Day 5

We will be using jupyter hub from https://lab2.pipal.in for this training.

create a notebook with name module3-day5

Debugging of python programs

• First would be look carefully at error trace given by python

x = 20
3x = 3*x # 3x is invalid variable name

  File "<ipython-input-4-acc217230c03>", line 2
    3x = 3*x # 3x is invalid variable name
     ^
SyntaxError: invalid syntax

3x # here 3 and x are seperate..3 is litteral integer 3 and x is some name of variable

  File "<ipython-input-3-1f391cb62d4c>", line 1
    3x # here 3 and x are seperate..3 is litteral integer 3 and x is some name of variable
     ^
SyntaxError: invalid syntax

x3 # x and 3 is not seperate 
x334 # valid variable name
x3-1 # this is not variable name..this is a statement...

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-5-7fd632fa0481> in <module>
----> 1 x3 # x and 3 is not seperate
      2 x334 # valid variable name
      3 x3-1 # this is not variable name..this is a statement...

NameError: name 'x3' is not defined

x = 20
x3 = 3*x
x334 = 334*x

x3 # x and 3 is not seperate 
x334 # valid variable name
x3-1 # this is not variable name..this is a statement...

59

x3_1 = x3 - 1

rules for variable names/function/class ..genrally any name

1. alphanumeric with _ as additiona char ...
2. name can not start with digit

twice_x = 2*x

def mysum(nums):
    s = 0
    for n in nums:
        s += n 
        return s # indentation of return is wrong
                 # this kind problem is difficult to find
                 # logic and lack understanding of sytax

# if all iterations are getting executed..then look at indentation of return 

def mysum(nums):
    s = 0
    for n in nums:
        s += n 
    return s 

def function2(data, i):
    return [item[i] for item in data]


def function(data):
    t = []
    n = len(data)
    for i in range(n):
        t.append(function2(data, i)
    return t
    
# if line shown by syntax error does not have any sytax error..mostly previous line has some bracket missing

  File "<ipython-input-17-3fca435bc11d>", line 10
    return t
    ^
SyntaxError: invalid syntax

def function2(data, i):
    return [item[i] for item in data]


def function(data):
    t = []
    n = len(data)
    for i in range(n):
        t.append(function2(data,i))
    return t
    

def foo():
    retun 5

  File "<ipython-input-19-98e6a4bc7044>", line 2
    retun 5
          ^
SyntaxError: invalid syntax

for i in range(5) # : is missing
   print(i)

  File "<ipython-input-21-cb7093f3c5b5>", line 1
    for i in range(5) # : is missing
                      ^
SyntaxError: invalid syntax

File "C:\Users\sanghvi\AppData\Local\Temp/ipykernel_2444/4184950771.py", line 27 def saveattachemnts_from_email(ivy_export-1634915516683.xlsb): ^ SyntaxError: invalid syntax

def saveattachemnts_from_email(filename):
    pass

saveattachemnts_from_email("ivy_export-1634915516683.xlsb")

saveattachemnts_from_email(ivy_export-1634915516683.xlsb)

  File "<ipython-input-26-778e7ea250a8>", line 1
    saveattachemnts_from_email(ivy_export-1634915516683.xlsb)
                                                        ^
SyntaxError: invalid syntax

def add(x, y):
    return x+y

add(2,3)

5

def add1(2, 3): # literals are not allowed as parameter!
    return 2+3

logical mistakes

def function2(data, i):
    return [item[i] for item in data]

# Question is what can be data? int, float, dictionary, list, tuple

def function2(data, i):
    return [item[i] for item in data] 


def function(data):
    t = []
    n = len(data)
    for i in range(n):
        t.append(function2(data,i))
    return t

# observations
# function
# 1. function returns a list of list
# 2. 

# function2
# 1. returns a list
# 

{1:[1, 2, 3, 4], 2:[2, 3, 4, 5]}

{1: [1, 2, 3, 4], 2: [2, 3, 4, 5]}

def column(data2d, colnum): # function2
    return [row[colnum] for row in data2d]

def transpose(data2d):
    transpose_ = []
    colcount = len(data2d[0])
    for c in range(colcount):
        transpose_.append(column(data2d, c))
    return transpose_

def transpose(data2d):
    colcount = len(data2d)
    return [column(data2d, c) for c in range(colcount)]

you = 45
me  = 4556
yo = 458098435
x1 = 343
x2 = 343

variable_names_are_big_thats_fine = 54

class InsufficientFluid(Exception):
    pass

class VendingMachine:
    
    def __init__(capacity):
        self.capacity = capacity
        self.present_volume = 0
        
    def dispense():
        if self.present_volume > 200:
            self.present_volume -= 200
        else:
            raise InsufficientFluid("Refill the machine , if you want to dispense beverage!")
    
    def refill(amount):
        if amount + self.present_volume > self.capacity:
            raise Exception(f"You can not fill beyond capacity {self.capacity}")
        else:
            self.present_volume += volume

v = VendingMachine(3000) # -> VendingMachine.__init__(self, 3000)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-38-4b2f911e3207> in <module>
----> 1 v = VendingMachine(3000)

TypeError: __init__() takes 1 positional argument but 2 were given

class InsufficientFluid(Exception):
    pass

class VendingMachine:
    
    def __init__(self, capacity):
        self.capacity = capacity
        self.present_volume = 0
        
    def dispense(self):
        if self.present_volume > 200:
            self.present_volume -= 200
        else:
            raise InsufficientFluid("Refill the machine , if you want to dispense beverage!")
    
    def refill(self, volume):
        if volume + self.present_volume > self.capacity:
            raise Exception(f"You can not fill beyond capacity {self.capacity}")
        else:
            self.present_volume += volume

v = VendingMachine(3000)

v.refill(500)

class Squared:
    
    def __init__(self, z):
        self.z = z
        
    def intermediate_method(self):
        return self.square()
    
    def square(self):
        return z*z

s = Squared(5)

s.intermediate_method()

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-54-33d1ea25b5ce> in <module>
----> 1 s.intermediate_method()

<ipython-input-52-62a0470d4e25> in intermediate_method(self)
      5 
      6     def intermediate_method(self):
----> 7         return self.square()
      8 
      9     def square(self):

<ipython-input-52-62a0470d4e25> in square(self)
      8 
      9     def square(self):
---> 10         return z*z

NameError: name 'z' is not defined

class Squared:
    
    def __init__(self, z):
        self.z = z
        
    def intermediate_method(self):
        return self.square()
    
    def square(self):
        return self.z*self.z

debugging using pdb

!ls *.py

aad.py		  bank0.py    download.py	 mymodule.py
addapp.py	  bank1.py    extract_tableA.py  outlook.py
add.py		  cat.py      head.py		 square.py
anotherpyfile.py  combine.py  mymodule1.py	 testmodule.py

!cat combine.py

import typer
import csv

app = typer.Typer()

def find_common_cols(file1, file2):
    with open(file1) as f1:
        csvf1 = csv.DictReader(f1)
        with open(file2) as f2:
            csvf2 = csv.DictReader(f2)
            row1 = next(csvf1)
            row2 = next(csvf2)
        return row1.keys() & row2.keys()
    

@app.command()
def combine_csvs(file1:str, file2:str, outputfile:str): # type annotation 
    commoncols = find_common_cols(file1, file2)
    with open(file1) as f1:
        csvf1 = csv.DictReader(f1)
        with open(file2) as f2:
            csvf2 = csv.DictReader(f2)
            with open(outputfile, "w") as w:
                csvfw = csv.DictWriter(w, commoncols)
                csvfw.writeheader()
                for row in csvf1:
                    csvfw.writerow({c:row[c] for c in commoncols})

                for row in csvf2:
                    csvfw.writerow({c:row[c] for c in commoncols})


if __name__ == "__main__":
    app()

!ls *.csv

A.csv		file1.csv	      myindexdata.csv  tables.csv
combined.csv	file2.csv	      mytables.csv     timeseries.csv
csvdata.csv	indexdata.csv	      output.csv       wallet.csv
data.csv	indexdata_quoted.csv  sum_by_cats.csv
empty_data.csv	index_write.csv       tableA.csv

!cat file1.csv

item,price,category,notes
amazon kindle book,545.0,books,some notes
mobile recharge,499.0,communication,sdksljakd
amazon kindle book1,550.0,books,some notes
data recharge,399.0,communication,sdksljakd

!cat file2.csv

item,price,category
air ticket,6000.0,travel
meal,499.0,food

Packaging your code as python package

lets create a package with name combinecsvs

combinecsvs
    |-setup.py
    |-requirements.txt
    + A
      |-__init__.py
      |- combine.py
      + B
        |-__init__.py

!mkdir combinecsvs

!mkdir combinecsvs/A

!mkdir combinecsvs/A/B

!touch combinecsvs/A/__init__.py

%%file combinecsvs/A/__init__.py
from . import combine

Overwriting combinecsvs/A/__init__.py

!mkdir combinecsvs/A/B/__init__.py

!cp combine.py combinecsvs/A/

%%file combinecsvs/setup.py
from distutils.core import setup

def requirements():
    with open("requirements.txt") as f:
        return [line.strip() for line in f]

setup(
    name="combinecsvs",
    version="1.0",
    description="A sample module which contains a pyscript to combine csvs",
    author="vikrant",
    author_email="vikran@dls.fdskjf.com",
    url="https://combinecsvs.com",
    packages=['A','A.B'],
    install_requires=requirements(),
    include_package_data=True,
    entry_points={"console_scripts":["combine_csv=A.combine:main"]},
)

Overwriting combinecsvs/setup.py

%%file combinecsvs/requirements.txt
typer

Writing combinecsvs/requirements.txt

%%file combinecsvs/A/combine.py
import csv
import sys


def find_common_cols(file1, file2):
    with open(file1) as f1:
        csvf1 = csv.DictReader(f1)
        with open(file2) as f2:
            csvf2 = csv.DictReader(f2)
            row1 = next(csvf1)
            row2 = next(csvf2)
        return row1.keys() & row2.keys()
    


def combine_csvs(file1:str, file2:str, outputfile:str): # type annotation 
    commoncols = find_common_cols(file1, file2)
    with open(file1) as f1:
        csvf1 = csv.DictReader(f1)
        with open(file2) as f2:
            csvf2 = csv.DictReader(f2)
            with open(outputfile, "w") as w:
                csvfw = csv.DictWriter(w, commoncols)
                csvfw.writeheader()
                for row in csvf1:
                    csvfw.writerow({c:row[c] for c in commoncols})

                for row in csvf2:
                    csvfw.writerow({c:row[c] for c in commoncols})

                    
def main():
    combine_csvs(sys.argv[1], sys.argv[2], sys.argv[3])

if __name__ == "__main__":
    main()

Overwriting combinecsvs/A/combine.py

typer does not go very well with entry points, but another module called click... it works