Basic Python Training at Grofers - Day 3 (Part 2)¶

Dec 19-21, 2019

Vikrant Patil
Anand Chitipothu

These notes are available online at http://notes.pipal.in/2019/grofers_basic_dec/

Day 1 | Day 2 | Day 3 | Day 3 - Part 2

Python Quiz¶

# p1
x = 1
y = x
x = 2
print(x, y)

2 1

# p2
x = [1, 2]
y = [x, 5]
x.append(3)
print(y)

[[1, 2, 3], 5]

x = 1
y = [x, 5]
x = 2
print(y)

[1, 5]

# p3
x = [1, 2]
y = [x, 5]
y[0].append(3)
print(x, y)

[1, 2, 3] [[1, 2, 3], 5]

# p4
x = [1, 2]
y = [x, 5]
y[0] = 3
print(x, y)

[1, 2] [3, 5]

# p5

def append(x):
    x.append(0)

numbers = [1, 2, 3]
append(numbers)
print(numbers)

[1, 2, 3, 0]

# p6

def incr(x):
    x = x + 1
    
n = 5
incr(n)
print(n)

5

Testing Web Applications using Selenium¶

Setup¶

Install Selenium Python Library

$ python3 -m pip install selenium

Download and Install Selenium Web Driver for Chome (or Firefox)

Download the appropriate version for your operating system and the browser version.
- Google Chrome Driver: https://sites.google.com/a/chromium.org/chromedriver/
- Firefox Driver: https://github.com/mozilla/geckodriver/releases
Latest version for Mac OS X:
```
$ wget https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_mac64.zip    
$ unzip chromedriver_mac64.zip
```
Latest version for Linux:
```
$ wget https://chromedriver.storage.googleapis.com/79.0.3945.36/chromedriver_linux64.zip    
$ unzip chromedriver_linux64.zip
```

Example: Price of a product on grofers.com¶

%%file price.py
from selenium import webdriver

with webdriver.Chrome("./chromedriver") as driver:
    driver.get("https://grofers.com/s/?q=onion")
    time.sleep(5)
    price = driver.find_element_by_css_selector(
        ".plp-product .plp-product__price--new").text
    print(price)

Writing price.py

!python price.py

₹109

Example: library to interact with grofers website¶

%%file grofers.py
from urllib.parse import urlencode

class Grofers:
    def __init__(self, driver):
        self.driver = driver
        
    def select_city(self, name):
        self.driver.get("https://grofers.com")
        if not self._has_location_overlay():
            self.driver.find_element_by_css_selector(".user-address").click()
        city = self._find_city(name)
        city.find_element_by_css_selector(".img-loader__placeholder").click()
        
    def _has_location_overlay(self):
        try:
            self.driver.find_element_by_css_selector(".location__overlay")
            return True
        # Should be NoSuchElementException
        except Exception:
            return False
        
    def _find_city(self, name):
        cities = self.driver.find_elements_by_css_selector(".cities-container-list__item")
        for city in cities:
            img = city.find_element_by_css_selector("img")
            if img.get_attribute("alt").lower() == name.lower(): 
                return city
        
    def search(self, q):
        url = "https://grofers.com/s/?" + urlencode({"q": q})
        self.driver.get(url)
        divs = self.driver.find_elements_by_css_selector(".plp-product")
        return [SearchResult(div) for div in divs]
    
    def get_cart_size(self):
        e = driver.find_element_by_css_selector(".shopping-cart .item-count").text
        return e.text
    
class SearchResult:
    def __init__(self, div):
        self.div = div
    
    def get_name(self):
        e = self.div.find_element_by_css_selector(".plp-product__name")
        return e.text
    
    def get_price(self):
        e = self.div.find_element_by_css_selector(".plp-product__price--new")
        return e.text
    
    def add_to_cart(self):
        self.div.find_element_by_css_selector(".add-to-cart__add-btn").click()

Overwriting grofers.py

%%file price_v2.py
from selenium import webdriver
from grofers import Grofers

with webdriver.Chrome("./chromedriver") as driver:
    g = Grofers(driver)
    results = g.search("toor dal")
    #for r in results:
    #    print(r.get_name(), r.get_price())
    results[0].add_to_cart()
    print(g.get_cart_size())

Overwriting price_v2.py

%%file onion_prices.py
from selenium import webdriver
from grofers import Grofers
import time

cities = ["Bengaluru", "Gurugram", "Kolkata", "Hyderabad", "New Delhi"]

with webdriver.Chrome("./chromedriver") as driver:
    g = Grofers(driver)
    for city in cities:
        g.select_city(city)
        time.sleep(5)
        results = g.search("onion")
        print(city, results[0].get_price())

Overwriting onion_prices.py

!python onion_prices.py

Bengaluru ₹109
Gurugram ₹109
Kolkata ₹145
Hyderabad ₹139
New Delhi ₹125

Problem: Write a program to search for a product and print the name, price and MRP for each of the match.

Locust¶

Install locust using:

$ pip install locustio

Sample locust file:

%%file locustfile.py

from locust import HttpLocust, TaskSet, task, between
import logging

logger = logging.getLogger()

class GrofersTaskSet(TaskSet):
    def on_start(self):
        """ on_start is called when a Locust start before any task is scheduled """
        logger.info("start")

    def on_stop(self):
        """ on_stop is called when the TaskSet is stopping """
        logger.info("stop")

    @task(10)
    def index(self):
        self.client.get("/")

    @task(3)
    def search(self):
        self.client.get("/search", params={"q": "onion"})

class GrofersUser(HttpLocust):
    task_set = GrofersTaskSet
    wait_time = between(1, 5)

Writing locustfile.py

The VM Setup¶

Available machines:

beta.docker.pipal.in
gamma.docker.pipal.in
delta.docker.pipal.in
epsilon.docker.pipal.in
zeta.docker.pipal.in
eta.docker.pipal.in
theta.docker.pipal.in
iota.docker.pipal.in
kappa.docker.pipal.in
lambda.docker.pipal.in
mu.docker.pipal.in
nu.docker.pipal.in
xi.docker.pipal.in
omicron.docker.pipal.in

$ ssh pipal@alpha.docker.pipal.in
password: vmware-docker

$ cd grofers
$ . venv/bin/activate

Run webserver using:

$ gunicorn -b :8080 webapp:app

Open a new ssh session and run:

$ locust

import requests
html = requests.get("https://grofers.com").text

from collections import Counter

words = html.split()
Counter(words).most_common(10)

[('&', 212),
 ('&amp;', 163),
 ('and', 65),
 ('<script', 51),
 ('class="img-loader__wrapper__wrapper"><div', 41),
 ('class="img-loader__wrapper"><img', 41),
 ('class="img-loader__img', 41),
 ('img-loader__img--hidden', 41),
 ('class="img-loader__placeholder', 41),
 ('img-loader__img', 41)]

from bs4 import BeautifulSoup

soup = BeautifulSoup(html)
words = soup.get_text().split()

Counter(words).most_common(20)

[('&', 292),
 ('and', 63),
 ('screen', 39),
 ('to', 26),
 ('=', 19),
 ('0', 16),
 ('Best', 13),
 ('Launches,', 12),
 ('{', 10),
 ('(max-width:1020px){.header', 9),
 ('(min-width:1020px)', 8),
 ('in', 8),
 ('Online', 7),
 ('-', 7),
 ('}', 7),
 ('Sauces', 7),
 ('Home', 7),
 ('Other', 7),
 ('Offers,', 7),
 ('Milk', 7)]